0 - Basic Information

In [1]:
from IPython.display import Image, display

display(Image(filename='files/Folie1.PNG'))
In [2]:
display(Image(filename='files/Folie2.PNG'))
In [3]:
display(Image(filename='files/Folie3.PNG'))
In [4]:
display(Image(filename='files/Folie4.PNG'))

1 - Basic Imports

In [5]:
#!/usr/bin/python
# -*- coding: utf-8 -*-

# IPython NoteBook created by Martin Schupfner, DKRZ
# Reference: Martin Juckes 2016 - 
#            dreqPy (Data Request Python API) & dreqPy User's Guide

from dreqPy import dreq

print("Using DreqPy (Data Request Python API) in version %s"\
    % str(dreq.version))

# Initialisation
dq = dreq.loadDreq()
Using DreqPy (Data Request Python API) in version 01.00.31.post3

2 - dq.coll Examples

In [6]:
# dq.coll
# Content Object dq.coll is a dictionary containing the data request sections,
#   with 3 elements (named tuples) for each section:
# - header :  named tuple with info such as title, lable, etc.
# - attDefn:  dictionary containing record attribute definitions
# - items  :  list of records

# Print all entries of dq.coll
print("dq.coll Entries:\n", ", ".join(dq.coll.keys()))
dq.coll Entries:
 __core__, __main__, __sect__, requestVarGroup, requestItem, exptgroup, miptable, CMORvar, objective, spatialShape, requestLink, tableSection, modelConfig, varChoiceLinkC, objectiveLink, remarks, experiment, requestVar, standardname, varChoiceLinkR, var, mip, varChoice, temporalShape, structure, grids, timeSlice, cellMethods, tags, varRelations, varRelLnk, qcranges, places, transfers, units
In [7]:
# header content (Example MIP Variable):
print(".header content Example 'var':\n------------------------------")

for name in dq.coll['var'].header._fields: print("%-15s : %s" \
    %(name, getattr(dq.coll['var'].header, name)))
.header content Example 'var':
------------------------------
tag             : table
label           : var
title           : 1.2 MIP Variable
id              : var
itemLabelMode   : def
level           : 0
maxOccurs       : 1
labUnique       : No
uid             : SECTION:var
description     : Each MIP variable record defines a MIP variable name, associated with a CF Standard Name.
In [8]:
# attDefn content (Example MIP Variable):
print(".attDefn content Example 'var':\n-------------------------------")

for key in dq.coll['var'].attDefn.keys(): 
    print("%-15s : %s" %(key, dq.coll['var'].attDefn[key]))
.attDefn content Example 'var':
-------------------------------
label           : Item <X.1 Core Attributes>: [label] Variable Name
title           : Item <X.1 Core Attributes>: [title] Long name
sn              : Item <X.1 Core Attributes>: [sn] CF Standard Name
units           : Item <X.1 Core Attributes>: [units] Units of Measure
description     : Item <X.1 Core Attributes>: [description] Record Description
procnote        : Item <X.1 Core Attributes>: [procnote] Processing Notes
procComment     : Item <X.1 Core Attributes>: [procComment] Processing Comments
prov            : Item <X.1 Core Attributes>: [prov] Notes on Provenance of Variable Specifications
uid             : Item <X.1 Core Attributes>: [uid] Record Identifier
provmip         : Item <X.1 Core Attributes>: [provmip] MIP Defining this Variable
unid            : Item <X.1 Core Attributes>: [unid] Link to Units section
In [9]:
# items content (Example MIP Variable):
print(".items content Example 'var':\n-----------------------------")

for key in dq.coll['var'].attDefn.keys(): 
    print("%-15s : %s" %(key, getattr(dq.coll['var'].items[0], key)))
.items content Example 'var':
-----------------------------
label           : abs550aer
title           : Ambient Aerosol Absorption Optical Thickness at 550nm
sn              : atmosphere_absorption_optical_thickness_due_to_ambient_aerosol_particles
units           : 1
description     : Optical thickness of atmospheric aerosols at wavelength 550 nanometers.
procnote        : ('',)
procComment     : 
prov            : CMIP5_aero
uid             : 0baf6a333b91c4db341b515c28cd2c05
provmip         : AerChemMIP
unid            : fd6fef50-3468-11e6-ba71-5404a60d96b5

3 - dq.inx Examples

In [10]:
# Index dq.inx is a simple lookup table
#   dq.inx.uid[UID] returns the record corresponding to 'UID'

#   Example from above:
item = dq.inx.uid['0baf6a333b91c4db341b515c28cd2c05']

print(item)
print("Label:", item.label)
print("Title:", item.title)
print("Units:", item.units)
Item <1.2 MIP Variable>: [abs550aer] Ambient Aerosol Absorption Optical Thickness at 550nm
Label: abs550aer
Title: Ambient Aerosol Absorption Optical Thickness at 550nm
Units: 1
In [11]:
#   dq.inx.iref_by_uid[UID]
#    gives a list of IDs of objects linked to 'UID',
#    as a tuple (SECT, ID)

#   Example from above:
id_list = dq.inx.iref_by_uid['0baf6a333b91c4db341b515c28cd2c05']

for ID in id_list:     
    print("#  Section: %-10s\n   UID: %15s\n   %s\n"\
    %(ID[0], ID[1], dq.inx.uid[ID[1]]))
#  Section: vid       
   UID: 19bebf2a-81b1-11e6-92de-ac72891c3257
   Item <1.3 CMOR Variable>: [abs550aer] Ambient Aerosol Absorption Optical Thickness at 550nm

In [12]:
#   dq.inx.iref_by_sect[UID].a[SECT] 
#    gives a list of IDs of 'SECT' linked to 'UID'

#   Example from above:
id_list = dq.inx.iref_by_sect['0baf6a333b91c4db341b515c28cd2c05'].a['CMORvar']

for ID in id_list: 
    item = dq.inx.uid[ID]
    print("%15s | %10s | %s" \
    %(item.label, item.mipTable, item.vid))
      abs550aer |     AERmon | 0baf6a333b91c4db341b515c28cd2c05
In [13]:
# The same can be achieved using dq.coll, though:

#   Example from above:
CMORvar_list = [ item for item in dq.coll['CMORvar'].items \
           if item.vid=='0baf6a333b91c4db341b515c28cd2c05']

for item in CMORvar_list:     
    print("%15s | %10s | %s" \
    %(item.label, item.mipTable, item.vid))
      abs550aer |     AERmon | 0baf6a333b91c4db341b515c28cd2c05

4 - Presentation of dq.coll Sections

Overview over all dq.coll sections and their items' attributes, as well as examples.

In [14]:
from IPython.display import Image, display

# Display Structure of DreqPy
display(Image(filename='files/DreqPyStructure.png'))
In [15]:
def showSect(sect, desc=False, uid=""):
    """
    Print the Content of dq.coll's sections and one example item
    Arg: sect - str     - section key
         desc - boolean - also print description 
                          (Default: False)
         uid  - str     - provide the uid of the example item if desired,
                          else, the example item will be chosen automatically
    """
    
    # print header title
    print(dq.coll[sect].header.title)
    print("-"*len(dq.coll[sect].header.title))
    
    # print Section name and description
    for subsect in dq.coll[sect].attDefn.keys():
        if desc==True:
            print("# %15s  |  %s%s" \
            %(subsect, dq.coll[sect].attDefn[subsect].title, \
            "\n"+dq.coll[sect].attDefn[subsect].description if \
            (str(dq.coll[sect].attDefn[subsect].description)!="" or \
            subsect in str(dq.coll[sect].attDefn[subsect].description)) \
            else ""))
        else:
            print("# %15s  |  %s" \
            %(subsect, dq.coll[sect].attDefn[subsect].title))
            
    # print Example of first item in list
    print("-------\nExample\n-------")
    for subsect in dq.coll[sect].attDefn.keys():
            if uid=="":
                print("# %15s  |  %s"\
                %(subsect, getattr(dq.coll[sect].items[0], subsect) if \
                 subsect not in str(getattr(dq.coll[sect].items[0], subsect)) \
                 else ""))
            else:
                print("# %15s  |  %s"\
                %(subsect, getattr(dq.inx.uid[uid], subsect) if \
                 subsect not in str(getattr(dq.coll[sect].items[0], subsect)) \
                 else ""))
In [16]:
# MIPs
print("Chicken or egg? MIP or Objective?\n"\
      "The CMIP endorsed MIP (Model Intercomparison Project) is linked to scientific objectives.")
display(Image(filename='files/DreqPyStructure_1.png'))
showSect('mip')
#showSect('mip', desc=True) #Activate for additional info if there is any
Chicken or egg? MIP or Objective?
The CMIP endorsed MIP (Model Intercomparison Project) is linked to scientific objectives.
1.1 Model Intercomparison Project
---------------------------------
#           label  |  MIP short name
#           title  |  MIP title
#             uid  |  Record identifier
#     description  |  Description of the Model Intercomparison Project
#             url  |  Project Home Page
-------
Example
-------
#           label  |  AerChemMIP
#           title  |  Aerosols and Chemistry Model Intercomparison Project
#             uid  |  AerChemMIP
#     description  |  
#             url  |  https://wiki.met.no/aerocom/aerchemmip/start
In [17]:
# Objective
showSect('objective')
#showSect('objective', desc=True) #Activate for additional info if there is any
1.6 Scientific objectives
-------------------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record identifier
#     description  |  Description
#             mip  |  Endorsed MIP
-------
Example
-------
#           label  |  aerChem
#           title  |  AerChemMIP diagnostics
#             uid  |  030af762-c570-11e8-8bae-a44cc8186c64
#     description  |  AerChemMIP diagnostics
#             mip  |  AerChemMIP
In [18]:
# Experiments
print("MIPs may define experiments or solely request data from experiments.")
print("Experiments are organised in experiment groups.")
display(Image(filename='files/DreqPyStructure_2.png'))
showSect('experiment')
#showSect('experiment', desc=True) #Activate for additional info if there is any
MIPs may define experiments or solely request data from experiments.
Experiments are organised in experiment groups.
1.5 Experiments
---------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record identifier
#     description  |  Description
#            egid  |  Identifier for experiment group
#             mip  |  MIP defining experiment
#            mcfg  |  Model Source Types
#            tier  |  Tier of Experiment
#          nstart  |  Number of Start Dates
#          starty  |  Start year
#            endy  |  End year
#             yps  |  Years per Simulation Including all Start Years
#            ensz  |  Ensemble size
#            ntot  |  Total number of years
#         comment  |  Comment
-------
Example
-------
#           label  |  1pctCO2
#           title  |  1 percent per year increase in CO2
#             uid  |  f16fc0b0-dd9e-11e6-b89b-ac72891c3257
#     description  |  Increase atmospheric CO2 concentration gradually at a rate of 1 percent per year. The concentration of atmospheric carbon dioxide is increased from the global annual mean 1850 value until quadrupling. 
#            egid  |  f16fbb42-dd9e-11e6-b89b-ac72891c3257
#             mip  |  CMIP
#            mcfg  |  AOGCM | AER CHEM BGC
#            tier  |  [1]
#          nstart  |  1
#          starty  |  
#            endy  |  
#             yps  |  150
#            ensz  |  [1]
#            ntot  |  150
#         comment  |  
In [19]:
# Experiment Groups
showSect('exptgroup')
#showSect('exptgroup', desc=True) #Activate for additional info if there is any
1.9 Experiment Group
--------------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record identifier
#         tierMin  |  Minimum tier of experiments in group
#            ntot  |  Total number of years
-------
Example
-------
#           label  |  Aerchemmip1
#           title  |  Perturbed historical experiments
#             uid  |  8709a634-51c3-11e7-9bc5-5404a60d96b5
#         tierMin  |  1
#            ntot  |  1185
In [20]:
# Request Item
print("The request items build up the data request.")
print("They are linked to (or: requested by) either MIPs, experiments or experiment groups.")
print("Request items hold information about the time slice of the requested variables they include\n"\
      "as well as the possibility to set their priority and tier.")
display(Image(filename='files/DreqPyStructure_3.png'))
showSect('requestItem', True)
#showSect('requestItem', desc=True) #Activate for additional info if there is any
The request items build up the data request.
They are linked to (or: requested by) either MIPs, experiments or experiment groups.
Request items hold information about the time slice of the requested variables they include
as well as the possibility to set their priority and tier.
3.2 Request Item: specifying the number of years for an experiment
------------------------------------------------------------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record Identifier
#             mip  |  The MIP making the request. 
Model Intecomparison Project associated with the requestItem. Redundant because this is specified through the requestLink.
#             tab  |  Redundant?
Redundant attribute ... 
#            expt  |  Name of experiment or group of experiments
This is redundant: the information is provided through the esid link.
#            rlid  |  Identifier of Corresponding Request Link
Link to a requestLink record, which makes the connection to a variableGroup and a set of objectives.
#            esid  |  A link to an experiment, an experiment group or a MIP
This links to an individual experiment, to an experimentGroup, specifying a collection of experiments, or to a MIP. If it links to a MIP it means that the request applies to all experiments defined by that MIP.
#     esidComment  |  Comment on experiment(s) linked to.
An explanatory commemt for the esid attribute.
#          preset  |  Option to override priority set in each variable group
If, for example, preset is set to 2, all priority one variables in the variable group associated with this request are treated as priority 2 variables.
#          treset  |  Option to override tier set for experiment(s)
If, for example, treset is set to 1, all tier 2 and 3 experiments associated with this request are treated as tier 1 variables.
#              ny  |  Default number of years.
Default number of years, only used if experiment specifications are incomplete: will be redundant in final request.
#          nexmax  |  Maximum number of experiments requested.
Used to provide volume estimate before the links to experiment groups was fully functional.
#          nenmax  |  Number of ensemble members requested. 
If set to -1 then the request applies to to all the ensemble members specified in the ensz attribute of the experiment or experiments associated with the request. Note that ensz is a default ensemble size, and nenmax may be greater if one MIP wants more than the default number of ensembles. 
#           nymax  |  Number of years requested.
Number of years specified by the requesting MIP (will be redundant when links to temporal slices are fully implemented).
#          tslice  |  Selection of years from experiment
Optional link to a time slice specifier which will define subset of the years from an experiment.
-------
Example
-------
#           label  |  1pctco2
#           title  |  SEAPODYM.1: 1pctco2
#             uid  |  07f7b74e-26b9-11e7-9d3d-ac72891c3257
#             mip  |  VIACSAB
#             tab  |  
#            expt  |  1pctCO2
#            rlid  |  07e99fba-26b9-11e7-9d3d-ac72891c3257
#            esid  |  f16fc0b0-dd9e-11e6-b89b-ac72891c3257
#     esidComment  |  
#          preset  |  -1
#          treset  |  
#              ny  |  160
#          nexmax  |  
#          nenmax  |  -1
#           nymax  |  -1.0
#          tslice  |  
In [21]:
# Time Slice
showSect('timeSlice')
#showSect('timeSlice', desc=True) #Activate for additional info if there is any
3.11 Time Slices for Output Requests
------------------------------------
#           label  |  Record Label
#           title  |  Record Title
#            type  |  Type of time slice
#           start  |  Start year
#             end  |  End year
#            step  |  Step (years)
#        sliceLen  |  Length of slice
#          nyears  |  Total number of years
#             uid  |  Unique identifier
#       startList  |  Optional list of start times.
#    sliceLenUnit  |  Units of slice length
#     description  |  Description
#           child  |  Child experiment
-------
Example
-------
#           label  |  abrupt20
#           title  |  Years 131 to 150 from start
#            type  |  relativeRange
#           start  |  131
#             end  |  150
#            step  |  
#        sliceLen  |  
#          nyears  |  20.0
#             uid  |  _slice_abrupt20
#       startList  |  
#    sliceLenUnit  |  
#     description  |  
#           child  |  
In [22]:
# Request Variable Group
print("The request variable groups are defined by MIPs.")
display(Image(filename='files/DreqPyStructure_4.png'))
showSect('requestVarGroup')
#showSect('requestVarGroup', desc=True) #Activate for additional info if there is any
The request variable groups are defined by MIPs.
3.1 Request variable group: a collection of request variables
-------------------------------------------------------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record Identifier
#             mip  |  Endorsed MIP defining the variable group
#             ref  |  Reference
#         refNote  |  Reference Note
-------
Example
-------
#           label  |  aer6hr
#           title  |  AerChemMIP 6 hourly data
#             uid  |  188bc360-c5ee-11e6-9285-ac72891c3257
#             mip  |  AerChemMIP
#             ref  |  new
#         refNote  |  AerChemMIP 6 hourly model levels
In [23]:
# Request Link
print("Each request item is linked (via request link) to a request variable group.")
print("Several request items can link to the same request variable group\n"\
      "for a different set of experiments, time slices, priorities, etc.")
print("Of course a request variable group can be requested by MIPs that did not define it.")
display(Image(filename='files/DreqPyStructure_5.png'))
showSect('requestLink',True)
#showSect('requestLink', desc=True) #Activate for additional info if there is any
Each request item is linked (via request link) to a request variable group.
Several request items can link to the same request variable group
for a different set of experiments, time slices, priorities, etc.
Of course a request variable group can be requested by MIPs that did not define it.
3.3 Request link: linking a set of variables and a set of experiments
---------------------------------------------------------------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record Identifier
#             mip  |  Endorsed MIP requesting the data
Link to record defining the MIP using this record.
#             tab  |  Redundant
A redundant attribute.
#       objective  |  Science objectives associated with this request
Every request for model ouput is linked to one or more objectives. The XML link is made via objectiveLink records, each of which associates one requestLink with one objective record.
#            grid  |  Grid options
Specified required or preferred (depending on the value of gridreq) horizontal grid. Options: native (for model grid), 1deg, 2deg, 8 to 25km (8km preferred, less than 25km required), 5 to 25km (5km preferred, less than 25km required), blank (no prerefence).  
#         gridreq  |  Grid option constraints
Is the grid specified by the grid attribute optional (yes) or (no), conditionally (no*1 -- used for ocean data, when native is required only from models using a regular grid)
#         comment  |  Comment
Comment on the requestLink record.
#             ref  |  Reference
Not used.
#         refNote  |  Note on reference
A comment on the provenance of the record.
#           refid  |  Reference to a request Variable Group
Link to the requestVarGroup record defining the variables associated with this requestLink.
#             opt  |  Option for selecting a subset of variables
Option for specifying that only a subset of the variables specified in the requestVarGroup should be used. This option is designed to enable the re-use of groups when an easily identified subset of an existing group is wanted. Forreen completed at the request compilation stage. If 'prioirty' is specified, then only variables with priority less than the value specified by 'opar' should be used.
#            opar  |  parameter associated with *opt*
Parameter associated with 'opar'. If 'opar' is 'priority' it should be set to '1', '2', or '3'.
-------
Example
-------
#           label  |  3hr
#           title  |  3hr
#             uid  |  ef12df16-5629-11e6-9079-ac72891c3257__00
#             mip  |  CMIP
#             tab  |  3hr
#       objective  |  Model Intercomparison
#            grid  |  1deg
#         gridreq  |  
#         comment  |  
#             ref  |  
#         refNote  |  
#           refid  |  ef12df16-5629-11e6-9079-ac72891c3257
#             opt  |  all
#            opar  |  
In [24]:
# Objective Link
print("Request items are linked to scientific objectives (via objective link and request link).")
display(Image(filename='files/DreqPyStructure_6.png'))
showSect('objectiveLink')
#showSect('objectiveLink', desc=True) #Activate for additional info if there is any
Request items are linked to scientific objectives (via objective link and request link).
3.7 Link between scientific objectives and requests
---------------------------------------------------
#           label  |  Record Label
#           title  |  Record Title
#             uid  |  Record identifier
#             oid  |  Identifier for a scientific objective
#             rid  |  Identifier for a request link
-------
Example
-------
#           label  |  aerChem
#           title  |  aerChem [AerChemMIP]
#             uid  |  061156dcbfd0c4894c682f3235fbc696702eaaff
#             oid  |  030af762-c570-11e8-8bae-a44cc8186c64
#             rid  |  ef12e39e-5629-11e6-9079-ac72891c3257__00
In [25]:
# Request Variable
print("A request variable group holds a list of request variables.")
print("Request variables are basically links to CMOR variables with an\n"\
      "additional information about the CMOR variables' priority.")
display(Image(filename='files/DreqPyStructure_7.png'))
showSect('requestVar')
#showSect('requestVar', desc=True) #Activate for additional info if there is any
A request variable group holds a list of request variables.
Request variables are basically links to CMOR variables with an
additional information about the CMOR variables' priority.