DreqPy (Data Request Python API)
CMIP Phase 6 Data Request Python Interface
IPython NoteBook created by Martin Schupfner, DKRZ
References:
Download this website.
Synopsis¶
0 - Basic Information
1 - Basic Imports
2 - dq.coll Examples
3 - dq.inx Examples
4 - Presentation of dq.coll sections
5 - Advanced Example
6a - Data Request Volume Estimate
6b - Data Request Volume Estimate (Legacy - outdated)
7 - Additional Functions
8 - Official Web Tools
In [1]:
from IPython.display import Image, display
display(Image(filename='files/Folie1.PNG'))
In [2]:
display(Image(filename='files/Folie2.PNG'))
In [3]:
display(Image(filename='files/Folie3.PNG'))
In [4]:
display(Image(filename='files/Folie4.PNG'))
In [5]:
#!/usr/bin/python
# -*- coding: utf-8 -*-
# IPython NoteBook created by Martin Schupfner, DKRZ
# Reference: Martin Juckes 2016 -
# dreqPy (Data Request Python API) & dreqPy User's Guide
from dreqPy import dreq
print("Using DreqPy (Data Request Python API) in version %s"\
% str(dreq.version))
# Initialisation
dq = dreq.loadDreq()
In [6]:
# dq.coll
# Content Object dq.coll is a dictionary containing the data request sections,
# with 3 elements (named tuples) for each section:
# - header : named tuple with info such as title, lable, etc.
# - attDefn: dictionary containing record attribute definitions
# - items : list of records
# Print all entries of dq.coll
print("dq.coll Entries:\n", ", ".join(dq.coll.keys()))
In [7]:
# header content (Example MIP Variable):
print(".header content Example 'var':\n------------------------------")
for name in dq.coll['var'].header._fields: print("%-15s : %s" \
%(name, getattr(dq.coll['var'].header, name)))
In [8]:
# attDefn content (Example MIP Variable):
print(".attDefn content Example 'var':\n-------------------------------")
for key in dq.coll['var'].attDefn.keys():
print("%-15s : %s" %(key, dq.coll['var'].attDefn[key]))
In [9]:
# items content (Example MIP Variable):
print(".items content Example 'var':\n-----------------------------")
for key in dq.coll['var'].attDefn.keys():
print("%-15s : %s" %(key, getattr(dq.coll['var'].items[0], key)))
In [10]:
# Index dq.inx is a simple lookup table
# dq.inx.uid[UID] returns the record corresponding to 'UID'
# Example from above:
item = dq.inx.uid['0baf6a333b91c4db341b515c28cd2c05']
print(item)
print("Label:", item.label)
print("Title:", item.title)
print("Units:", item.units)
In [11]:
# dq.inx.iref_by_uid[UID]
# gives a list of IDs of objects linked to 'UID',
# as a tuple (SECT, ID)
# Example from above:
id_list = dq.inx.iref_by_uid['0baf6a333b91c4db341b515c28cd2c05']
for ID in id_list:
print("# Section: %-10s\n UID: %15s\n %s\n"\
%(ID[0], ID[1], dq.inx.uid[ID[1]]))
In [12]:
# dq.inx.iref_by_sect[UID].a[SECT]
# gives a list of IDs of 'SECT' linked to 'UID'
# Example from above:
id_list = dq.inx.iref_by_sect['0baf6a333b91c4db341b515c28cd2c05'].a['CMORvar']
for ID in id_list:
item = dq.inx.uid[ID]
print("%15s | %10s | %s" \
%(item.label, item.mipTable, item.vid))
In [13]:
# The same can be achieved using dq.coll, though:
# Example from above:
CMORvar_list = [ item for item in dq.coll['CMORvar'].items \
if item.vid=='0baf6a333b91c4db341b515c28cd2c05']
for item in CMORvar_list:
print("%15s | %10s | %s" \
%(item.label, item.mipTable, item.vid))
In [14]:
from IPython.display import Image, display
# Display Structure of DreqPy
display(Image(filename='files/DreqPyStructure.png'))
In [15]:
def showSect(sect, desc=False, uid=""):
"""
Print the Content of dq.coll's sections and one example item
Arg: sect - str - section key
desc - boolean - also print description
(Default: False)
uid - str - provide the uid of the example item if desired,
else, the example item will be chosen automatically
"""
# print header title
print(dq.coll[sect].header.title)
print("-"*len(dq.coll[sect].header.title))
# print Section name and description
for subsect in dq.coll[sect].attDefn.keys():
if desc==True:
print("# %15s | %s%s" \
%(subsect, dq.coll[sect].attDefn[subsect].title, \
"\n"+dq.coll[sect].attDefn[subsect].description if \
(str(dq.coll[sect].attDefn[subsect].description)!="" or \
subsect in str(dq.coll[sect].attDefn[subsect].description)) \
else ""))
else:
print("# %15s | %s" \
%(subsect, dq.coll[sect].attDefn[subsect].title))
# print Example of first item in list
print("-------\nExample\n-------")
for subsect in dq.coll[sect].attDefn.keys():
if uid=="":
print("# %15s | %s"\
%(subsect, getattr(dq.coll[sect].items[0], subsect) if \
subsect not in str(getattr(dq.coll[sect].items[0], subsect)) \
else ""))
else:
print("# %15s | %s"\
%(subsect, getattr(dq.inx.uid[uid], subsect) if \
subsect not in str(getattr(dq.coll[sect].items[0], subsect)) \
else ""))
In [16]:
# MIPs
print("Chicken or egg? MIP or Objective?\n"\
"The CMIP endorsed MIP (Model Intercomparison Project) is linked to scientific objectives.")
display(Image(filename='files/DreqPyStructure_1.png'))
showSect('mip')
#showSect('mip', desc=True) #Activate for additional info if there is any
In [17]:
# Objective
showSect('objective')
#showSect('objective', desc=True) #Activate for additional info if there is any
In [18]:
# Experiments
print("MIPs may define experiments or solely request data from experiments.")
print("Experiments are organised in experiment groups.")
display(Image(filename='files/DreqPyStructure_2.png'))
showSect('experiment')
#showSect('experiment', desc=True) #Activate for additional info if there is any
In [19]:
# Experiment Groups
showSect('exptgroup')
#showSect('exptgroup', desc=True) #Activate for additional info if there is any
In [20]:
# Request Item
print("The request items build up the data request.")
print("They are linked to (or: requested by) either MIPs, experiments or experiment groups.")
print("Request items hold information about the time slice of the requested variables they include\n"\
"as well as the possibility to set their priority and tier.")
display(Image(filename='files/DreqPyStructure_3.png'))
showSect('requestItem', True)
#showSect('requestItem', desc=True) #Activate for additional info if there is any
In [21]:
# Time Slice
showSect('timeSlice')
#showSect('timeSlice', desc=True) #Activate for additional info if there is any
In [22]:
# Request Variable Group
print("The request variable groups are defined by MIPs.")
display(Image(filename='files/DreqPyStructure_4.png'))
showSect('requestVarGroup')
#showSect('requestVarGroup', desc=True) #Activate for additional info if there is any
In [23]:
# Request Link
print("Each request item is linked (via request link) to a request variable group.")
print("Several request items can link to the same request variable group\n"\
"for a different set of experiments, time slices, priorities, etc.")
print("Of course a request variable group can be requested by MIPs that did not define it.")
display(Image(filename='files/DreqPyStructure_5.png'))
showSect('requestLink',True)
#showSect('requestLink', desc=True) #Activate for additional info if there is any
In [24]:
# Objective Link
print("Request items are linked to scientific objectives (via objective link and request link).")
display(Image(filename='files/DreqPyStructure_6.png'))
showSect('objectiveLink')
#showSect('objectiveLink', desc=True) #Activate for additional info if there is any
In [25]:
# Request Variable
print("A request variable group holds a list of request variables.")
print("Request variables are basically links to CMOR variables with an\n"\
"additional information about the CMOR variables' priority.")
display(Image(filename='files/DreqPyStructure_7.png'))
showSect('requestVar')
#showSect('requestVar', desc=True) #Activate for additional info if there is any