Add project reconstruction feature

docs/requirements-docs.txt

@@ -1 +1 @@
-furo
+sphinx_rtd_theme

docs/source/conf.py

@@ -13,7 +13,7 @@
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = []
+extensions = ['sphinx_rtd_theme']
 
 templates_path = ['_templates']
 exclude_patterns = []
@@ -23,6 +23,6 @@
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'furo'
+html_theme = 'sphinx_rtd_theme'
 html_logo = 'scanosslogo.jpg'
 html_static_path = ['_static']

inc/file.h

@@ -12,5 +12,5 @@ bool is_dir(char *path);
 void get_file_md5(char *filepath, uint8_t *md5_result);
 bool count_all_files(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *raw_data, uint32_t datalen, int iteration, void *ptr);
 char *get_file_extension(uint8_t *md5);
-
+char * path_query(uint8_t * file_id);
 #endif

inc/limits.h

@@ -37,9 +37,9 @@
 
 /* Snippets */
 #define DEFAULT_MATCHMAP_FILES 10000     // Default number of files evaluated in snippet matching
-#define MAX_MATCHMAP_FILES (DEFAULT_MATCHMAP_FILES * 5)     // Max number of files evaluated in snippet matching to prevent performance issues
+#define MAX_MATCHMAP_FILES (DEFAULT_MATCHMAP_FILES * 10)     // Max number of files evaluated in snippet matching to prevent performance issues
+#define MIN_LINES_COVERAGE 0.8
 #define SKIP_SNIPPETS_IF_FILE_BIGGER (1024 * 1024 * 4)
-#define SKIP_SNIPPETS_IF_STARTS_WITH (const char*[3]) {"{", "<?xml", "<html"}
 #define MAX_SNIPPETS_SCANNED 2500
 
 /* Variables */

inc/match.h

@@ -13,6 +13,7 @@ typedef struct match_data_t
 	component_list_t component_list; /*Component list object */ 
 	match_t type; /*math type (none, snippet, file) */
     int hits; /*match hits number, more hits equal bigger snippet matching*/
+	int lines_matched; /*number of matched lines*/
 	char * line_ranges; /*input snippet line ranges */
 	char * oss_ranges; /* kb snippet line ranges */
 	char * matched_percent; /* matched percent */
@@ -34,6 +35,4 @@ void output_matches_json(scan_data_t *scan);
 void compile_matches(scan_data_t *scan);
 match_list_t * match_select_m_best(scan_data_t * scan);
 match_list_t * match_select_m_component_best(scan_data_t * scan);
-bool component_from_file(struct ldb_table * table, uint8_t *key, uint8_t *subkey, uint8_t *raw_data, uint32_t datalen, int iteration, void *ptr);
-
 #endif

inc/match_list.h

@@ -78,7 +78,8 @@
 #define SCAN_MAX_SNIPPETS_DEFAULT 	1
 #define SCAN_MAX_COMPONENTS_DEFAULT 3
 
-#define MATCH_LIST_TOLERANCE 95
+#define MATCH_LIST_TOLERANCE 99.9
+
 typedef struct match_data_t match_data_t; /* Forward declaration */
 
 typedef enum
@@ -153,6 +154,7 @@ void component_list_print(component_list_t * list, bool (*printer) (component_da
 void component_list_destroy(component_list_t *list);
 bool component_list_add_binary(component_list_t *list, component_data_t *new_comp, bool (*val)(component_data_t *a, component_data_t *b), bool remove_a);
 bool match_list_eval(match_list_t *list, match_data_t * in,  bool (*eval)(match_data_t *fpa, match_data_t *fpb));
+void match_list_tolerance_set(float in);
 list_update_t component_list_update(component_list_t *list, component_data_t * in, list_update_t (*eval)(component_data_t *fpa, component_data_t *fpb));
 void component_list_sort(struct comp_entry *np, bool (*val)(component_data_t *a, component_data_t *b));
 

inc/scanoss.h

@@ -38,12 +38,11 @@
 
 #define WFP_LN 4
 /* Log files */
-#define SCANOSS_VERSION "5.4.9"
+#define SCANOSS_VERSION "5.4.10"
 #define SCAN_LOG "/tmp/scanoss_scan.log"
 #define MAP_DUMP "/tmp/scanoss_map.dump"
 #define SLOW_QUERY_LOG "/tmp/scanoss_slow_query.log"
 
-#define API_URL "https://api.osskb.org"
 #define DEFAULT_OSS_DB_NAME "oss"
 
 /* Engine configuration flags */
@@ -123,6 +122,7 @@ extern char * component_hint;
 
 /* DB tables */
 extern struct ldb_table oss_url;
+extern struct ldb_table oss_pivot;
 extern struct ldb_table oss_file;
 extern struct ldb_table oss_path;
 extern struct ldb_table oss_wfp;
@@ -157,5 +157,6 @@ void ldb_get_first_record(struct ldb_table table, uint8_t* key, void *void_ptr);
 int binary_scan(char * bfp);
 
 uint32_t fetch_recordset(struct ldb_table table, uint8_t *key, ldb_record_handler_t handler, void *ptr);
+void get_project_files(char * url_key);
 
 #endif

src/component.c

@@ -286,7 +286,7 @@ bool fill_component(component_data_t *component, uint8_t *url_key, char *file_pa
 	{
 		component->purls[0] = strdup(purl);
 		component->purls_md5[0] = malloc(MD5_LEN);
-		MD5((uint8_t *)component->purls[0], strlen(component->purls[0]), component->purls_md5[0]);
+		oss_purl.hash_calc( (unsigned char *) component->purls[0], strlen(component->purls[0]), component->purls_md5[0]);
 	}
 	component->age = -1;
 	return true;

src/debug.c

@@ -210,7 +210,7 @@ void scan_benchmark()
 			scan->hashes[i] = rand() % 256 + (rand() % 256) * 256 + (rand() % 256) * 256 * 256 + (rand() % 256) * 256 * 256 * 256;
 		}
 		scan->hash_count = total_hashes;
-
+		scan->total_lines = 10;
 		ldb_scan_snippets(scan);
 		scan_data_free(scan);
 	}

src/file.c

@@ -260,3 +260,32 @@ char *get_file_extension(uint8_t *md5)
 	fetch_recordset(oss_file, md5, get_first_file, out);
 	return out;
 }
+
+static bool path_query_handler(struct ldb_table * table, uint8_t * key, uint8_t * subkey, uint8_t * data, uint32_t datalen, int record_number, void * ptr)
+{
+	char **path = ptr;
+	/* Decrypt data */
+	char * decrypted = decrypt_data(data, datalen, *table, key, subkey);
+	if (!decrypted || !*decrypted)
+		return false;
+
+	*path = decrypted;
+	return true;
+}
+/**
+ * @brief Get the file path from the path table.
+ * @param md5 input path md5
+ * @return string with the path
+ */
+char * path_query(uint8_t * file_id)
+{
+	char * path = NULL;
+	if (!path_table_present)
+	{
+		scanlog("path_query: path table must be present to use this query\n");
+		return NULL;
+	}
+
+	fetch_recordset(oss_path, file_id, path_query_handler, (void *) &path);
+	return path;
+}

src/help.c

@@ -38,10 +38,10 @@
  */
 void help ()
 {
-	printf ("ScanOSS Engine v%s\n", SCANOSS_VERSION);
-	printf ("\n\
-This program performs an OSS inventory for the given TARGET comparing against the ScanOSS Knowledgebase.\n\
-Results are printed in STDOUT in JSON format\n\
+  printf ("ScanOSS Engine v%s\n", SCANOSS_VERSION);
+  printf ("\n\
+This program performs an OSS inventory scan of the specified TARGET by comparing it against the ScanOSS Knowledgebase.\n\
+Results are displayed in JSON format through STDOUT.\n\
 \n\
 Syntax: scanoss [parameters] [TARGET]\n\
 \n\
@@ -56,22 +56,23 @@ Configuration:\n\
 -c HINT    Provide a component HINT to influence scan results.\n\
 -k KEY     Displays contents of file KEY from MZ sources archive.\n\
 -l LICENSE Displays OSADL metadata for the provided SPDX license ID.\n\
+-p URL_HASH Returns a list with the md5 and path for each project file (pivot table is requeried).\n\
 \n\
 Options:\n\
--t  Tests engine performance.\n\
--v  Display version and exit.\n\
--n  Specify DB name (default: oss).\n\
--h  Display this help and exit.\n\
--d  Save debugging information to disk (/tmp).\n\
--q  Produces no JSON output. Only debugging info via STDERR.\n\
+-t  Run engine performance tests.\n\
+-v  Show version information and exit.\n\
+-n  Set database name (default: oss).\n\
+-h  Display this help information and exit.\n\
+-d  Store debugging information to disk (/tmp).\n\
+-q  Suppress JSON output (show only debugging info via STDERR).\n\
 \n\
-Enviroment variables:\n\
-SCANOSS_MATCHMAP_MAX: define the snippet scanning match map size, %d by default.\n\
-SCANOSS_API_URL: defines the API url, %s by default.\n\
+Environment variables:\n\
+SCANOSS_MATCHMAP_MAX: Set the snippet scanning match map size (default: %d).\n\
+SCANOSS_FILE_CONTENTS_URL: Define the API URL endpoint for sources. Source url wont be reported if it's not defined.\n\
 \n\
 Engine scanning flags:\n\
-The scanning engine can be configured by passing configuration flags with the -F parameter.\n\
-Alternatively, these value can be written in %s\n\
+Configure the scanning engine using flags with the -F parameter.\n\
+These settings can also be specified in %s\n\
 +-------+-------------------------------------------------------+\n\
 | Flag  | Setting                                               |\n\
 +-------+-------------------------------------------------------+\n\
@@ -82,7 +83,7 @@ Alternatively, these value can be written in %s\n\
 |   16  | Disable copyrights (default: enabled)                 |\n\
 |   32  | Disable vulnerabilities (default: enabled)            |\n\
 |   64  | Disable quality (default: enabled)                    |\n\
-|  128  | Disable cryptography (defalt: enabled)                |\n\
+|  128  | Disable cryptography (default: enabled)               |\n\
 |  256  | Disable best match only (default: enabled)            |\n\
 |  512  | Hide identified files (default: disabled)             |\n\
 | 1024  | Enable download_url (default: disabled)               |\n\
@@ -91,8 +92,7 @@ Alternatively, these value can be written in %s\n\
 | 8192  | Disable health layer (default: enabled)               |\n\
 | 16384 | Enable high accuracy, slower scan (default: disabled) |\n\
 +-------+-------------------------------------------------------+\n\
-Example: scanoss -F 12 DIRECTORY (scans DIRECTORY disabling license and dependency data)\n\
+Example: scanoss -F 12 DIRECTORY (scan DIRECTORY without license and dependency data)\n\
 \n\
-Copyright (C) 2018-2022 SCANOSS.COM\n", DEFAULT_MATCHMAP_FILES, API_URL, ENGINE_FLAGS_FILE);
-
+Copyright (C) 2018-2022 SCANOSS.COM\n", DEFAULT_MATCHMAP_FILES, ENGINE_FLAGS_FILE);
 }

src/main.c

@@ -48,6 +48,7 @@
 #include <dlfcn.h>
 
 struct ldb_table oss_url;
+struct ldb_table oss_pivot;
 struct ldb_table oss_file;
 struct ldb_table oss_path;
 struct ldb_table oss_wfp;
@@ -199,6 +200,10 @@ void initialize_ldb_tables(char *name)
 	oss_notices = ldb_read_cfg(dbtable);
 	oss_notices.hash_calc = hash_function_select(oss_notices.key_ln);
 
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "pivot");
+	oss_pivot = ldb_read_cfg(dbtable);
+	oss_pivot.hash_calc = hash_function_select(oss_pivot.key_ln);
+
 	kb_version_get();
 	osadl_load_file();
 
@@ -322,7 +327,7 @@ int main(int argc, char **argv)
 	int option;
 	bool invalid_argument = false;
 	char * ldb_db_name = NULL;
-	while ((option = getopt(argc, argv, ":f:s:b:B:c:k:a:F:l:n:M:N:wtvhdqH")) != -1)
+	while ((option = getopt(argc, argv, ":p:T:s:b:B:c:k:a:F:l:n:M:N:wtvhedqH")) != -1)
 	{
 		/* Check valid alpha is entered */
 		if (optarg)
@@ -386,6 +391,9 @@ int main(int argc, char **argv)
 			case 'N':
 				scan_max_components = atol(optarg);
 				break;
+			case 'T':
+				match_list_tolerance_set(atof(optarg));
+				break;
 			case 'w':
 				force_wfp = true;
 				break;
@@ -398,6 +406,11 @@ int main(int argc, char **argv)
 				scan_benchmark();
 				exit(EXIT_SUCCESS);
 				break;
+			case 'p':
+				initialize_ldb_tables(ldb_db_name);
+				get_project_files(optarg);
+				exit(EXIT_SUCCESS);
+				break;
 
 			case 'v':
 				printf("scanoss-%s\n", SCANOSS_VERSION);

src/match.c

@@ -324,7 +324,7 @@ static bool component_hint_date_comparation(component_data_t *a, component_data_
 	if (!*a->release_date)
 		return true;
 
-	if (!path_is_third_party(a->file) && path_is_third_party(b->file) && !(engine_flags & ENABLE_PATH_HINT))
+	if (!path_is_third_party(a->file) && path_is_third_party(b->file))
 	{
 		scanlog("Component rejected by third party filter\n");
 		return false;
@@ -335,7 +335,7 @@ static bool component_hint_date_comparation(component_data_t *a, component_data_
 	{	
 		if (purl_source_check(a) > purl_source_check(b))
 		{
-			scanlog("Component prefered by vsource\n");
+			scanlog("Component prefered by source\n");
 			return true;
 		}
 
@@ -436,24 +436,6 @@ bool add_component_from_urlid(component_list_t *component_list, uint8_t *url_id,
 	return true;
 }
 
-bool path_query_handler(struct ldb_table * table, uint8_t * key, uint8_t * subkey, uint8_t * data, uint32_t datalen, int record_number, void * ptr)
-{
-	char **path = ptr;
-	/* Decrypt data */
-	char * decrypted = decrypt_data(data, datalen, *table, key, subkey);
-	if (!decrypted || !*decrypted)
-		return false;
-
-	*path = decrypted;
-	return true;
-}
-static char * path_query(uint8_t * file_id)
-{
-	char * path = NULL;
-	fetch_recordset(oss_path, file_id, path_query_handler, (void *) &path);
-	return path;
-}
-
 /**
  * @brief Load componentes for a match processing the file recordset list.
  * For each file in the recordset we will query for the oldest url in the url table.
@@ -517,41 +499,14 @@ bool component_from_file(struct ldb_table * table, uint8_t *key, uint8_t *subkey
 bool load_matches(match_data_t *match)
 {
 	scanlog("Load matches\n");
-
-	/* Compile match ranges and fill up matched percent */
-	int hits = 100;
-	int matched_percent = 100;
-
-	/* Get matching line ranges (snippet match) */
-	if (match->type == MATCH_SNIPPET)
-	{
-				scanlog("compile_ranges returns %d hits\n", hits);
-
-	/*	hits = compile_ranges(match);
-
-		if (hits < min_match_hits)
-		{
-			match->type = MATCH_NONE;
-			return false;
-		}
-
-		float percent = (hits * 100) / match->scan_ower->total_lines;
-		if (hits)
-			matched_percent = floor(percent);
-		if (matched_percent > 99)
-			matched_percent = 99;
-		if (matched_percent < 1)
-			matched_percent = 1;
-
-		asprintf(&match->matched_percent, "%u%%", matched_percent);*/
-	}
-	else if (match->type == MATCH_BINARY)
+
+	if (match->type == MATCH_BINARY)
 	{
 		asprintf(&match->line_ranges, "n/a");
 		asprintf(&match->oss_ranges, "n/a");
 		asprintf(&match->matched_percent, "%d functions matched", match->hits);
 	}
-	else
+	else if (match->type == MATCH_FILE)
 	{
 		asprintf(&match->line_ranges, "all");
 		asprintf(&match->oss_ranges, "all");
@@ -757,11 +712,13 @@ void match_select_best(scan_data_t *scan)
 				break;
 			}
 
-			if (!best_match_component->identified && match_component->identified)
+			if ((!best_match_component->identified && match_component->identified) ||
+				(strcmp(best_match_component->vendor,best_match_component->component) && !strcmp(match_component->vendor, match_component->component)) ||
+				(path_is_third_party(best_match_component->file) && !path_is_third_party(match_component->file)))
 			{
 				scanlog("Replacing best match for a prefered component\n");
 				scan->matches_list_array[i]->best_match = item->match;
-			}	
+			}
 		}
 	}