Merge branch 'end_to_end' of https://github.com/HYPERNETS/hypernets_processor into include_plots

� Conflicts:
�	hypernets_processor/data_io/dataset_util.py
�	hypernets_processor/data_io/format/variables.py
�	hypernets_processor/etc/processor.config
�	hypernets_processor/main_processor.py
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L1A_IRR_v0.0.nc
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L1A_RAD_v0.0.nc
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L1B_IRR_v0.0.nc
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L1B_RAD_v0.0.nc
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L1B_v0.0.nc
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L1D_v0.0.nc
�	hypernets_processor/out/BSBE/2020/3/12/HYPERNETS_W_VFFR_L2A_REF_v0.0.nc
�	hypernets_processor/test.txt

Author: pdevis

docs/atbd/Hypernets-NPL-Algorithm_Theoretical_Basis_Document-v0.0.docx

@@ -241,10 +241,20 @@ def preprocess_l0(self, datasetl0, datasetl1a):
 
         flagval = 2 ** (self.context.get_config_value("outliers"))
 
+        print(np.where(mask > 0))
+
+        break
+        datasetl0["quality_flag"].values = [
+            flagval + datasetl0["quality_flag"].values[i] if mask[i] == 1 else
+            datasetl0["quality_flag"].values[i] for i in range(len(mask))]
+
+
+
         datasetl0["quality_flag"].values = [
             flagval + datasetl0["quality_flag"].values[i] if mask[i] == 1 else
             datasetl0["quality_flag"].values[i] for i in range(len(mask))]
 
+
         datasetl1a["quality_flag"].values = [
             flagval + datasetl1a["quality_flag"].values[i] if mask[i] == 1 else
             datasetl1a["quality_flag"].values[i] for i in range(len(mask))]

docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.6.docx

@@ -4,7 +4,7 @@
 
 from hypernets_processor.version import __version__
 import string
-from xarray import Variable, DataArray, Dataset
+from xarray import Variable, DataArray
 import numpy as np
 
 
@@ -224,13 +224,10 @@ def get_default_fill_value(dtype):
     def _get_flag_encoding(da):
         """
         Returns flag encoding for flag type data array
-
         :type da: xarray.DataArray
         :param da: data array
-
         :return: flag meanings
         :rtype: list
-
         :return: flag masks
         :rtype: list
         """
@@ -247,10 +244,8 @@ def _get_flag_encoding(da):
     def unpack_flags(da):
         """
         Breaks down flag data array into dataset of boolean masks for each flag
-
         :type da: xarray.DataArray
         :param da: dataset
-
         :return: flag masks
         :rtype: xarray.Dataset
         """
@@ -268,13 +263,10 @@ def unpack_flags(da):
     def set_flag(da, flag_name, error_if_set=False):
         """
         Sets named flag for elements in data array
-
         :type da: xarray.DataArray
         :param da: dataset
-
         :type flag_name: str
         :param flag_name: name of flag to set
-
         :type error_if_set: bool
         :param error_if_set: raises error if chosen flag is already set for any element
         """
@@ -295,13 +287,10 @@ def set_flag(da, flag_name, error_if_set=False):
     def unset_flag(da, flag_name, error_if_unset=False):
         """
         Unsets named flag for specified index of dataset variable
-
         :type da: xarray.DataArray
         :param da: data array
-
         :type flag_name: str
         :param flag_name: name of flag to unset
-
         :type error_if_unset: bool
         :param error_if_unset: raises error if chosen flag is already set at specified index
         """
@@ -322,10 +311,8 @@ def unset_flag(da, flag_name, error_if_unset=False):
     def get_set_flags(da):
         """
         Return list of set flags for single element data array
-
         :type da: xarray.DataArray
         :param da: single element data array
-
         :return: set flags
         :rtype: list
         """
@@ -346,13 +333,10 @@ def get_set_flags(da):
     def check_flag_set(da, flag_name):
         """
         Returns if flag for single element data array
-
         :type da: xarray.DataArray
         :param da: single element data array
-
         :type flag_name: str
         :param flag_name: name of flag to set
-
         :return: set flags
         :rtype: list
         """
@@ -367,5 +351,7 @@ def check_flag_set(da, flag_name):
         return False
 
 
+
+
 if __name__ == "__main__":
     pass

hypernets_processor/calibration/calibrate.py

@@ -1,4 +1,6 @@
-FLAG_MEANINGS = ["saturation", "nonlinearity", "bad_pointing", "placeholder1", "lon_default", "lat_default", "outliers",
-                 "angles_missing", "lu_eq_missing", "fresnel_angle_missing", "fresnel_default", "temp_variability_ed",
+FLAG_COMMON = ["saturation", "nonlinearity", "bad_pointing", "placeholder1", "lon_default", "lat_default", "outliers"]
+
+FLAG_WATER = ["angles_missing", "lu_eq_missing", "fresnel_angle_missing", "fresnel_default", "temp_variability_ed",
                  "temp_variability_lu", "min_nbred", "min_nbrlu", "min_nbrlsky", "def_wind_flag", "simil_fail"]
 
+FLAG_LAND = []