get_target_sequence also checks anomaly db for failed jobs

Author: shunt16

hypernets_processor/main/sequence_processor_main.py

@@ -48,18 +48,27 @@ def get_target_sequences(context, to_archive):
     # If adding to archive, remove previously processed paths from list by referencing
     # archive db
 
-    if to_archive:
+    if to_archive is True:
         processed_products = [
             product["raw_product_name"]
             for product in context.archive_db["products"].find(
                 site=context.get_config_value("site")
             )
         ]
 
+        failed_products = [
+            product["raw_product_name"]
+            for product in context.anomaly_db["anomalies"].find(
+                site=context.get_config_value("site")
+            )
+        ]
+
+        complete_products = processed_products + failed_products
+
         directory = os.path.dirname(raw_paths[0])
 
         raw_products = [os.path.basename(raw_path) for raw_path in raw_paths]
-        raw_products = list(set(raw_products) - set(processed_products))
+        raw_products = list(set(raw_products) - set(complete_products))
         raw_paths = [
             os.path.join(directory, raw_product) for raw_product in raw_products
         ]

hypernets_processor/main/tests/test_sequence_processor_main.py

@@ -68,6 +68,7 @@ def test_get_target_sequences_nottoarchive(self):
         )
 
         expected_sequences = [
+            os.path.join(tmpdir, "data", "SEQ20200311T112230"),
             os.path.join(tmpdir, "data", "SEQ20200311T112330"),
             os.path.join(tmpdir, "data", "SEQ20200311T112430"),
             os.path.join(tmpdir, "data", "SEQ20200311T112530")

hypernets_processor/test/test_functions.py

@@ -62,7 +62,7 @@ def setup_test_anomaly_db(url):
     db["anomalies"].insert(
         dict(
             anomaly="anomaly_name",
-            raw_product_name="SEQ20200311T112430",
+            raw_product_name="SEQ20200311T112330",
             site="site",
         )
     )