Retrieval eval incomplete

README.md

@@ -1,12 +1,12 @@
 [![Coverage](https://img.shields.io/badge/coverage-95%25-green)](https://ontolearn-docs-dice-group.netlify.app/usage/09_further_resources#code-coverage)
-[![Pypi](https://img.shields.io/badge/pypi-0.7.3-blue)](https://pypi.org/project/ontolearn/0.7.3/)
-[![Docs](https://img.shields.io/badge/documentation-0.7.3-yellow)](https://ontolearn-docs-dice-group.netlify.app/usage/01_introduction)
+[![Pypi](https://img.shields.io/badge/pypi-0.7.4-blue)](https://pypi.org/project/ontolearn/0.7.4/)
+[![Docs](https://img.shields.io/badge/documentation-0.7.4-yellow)](https://ontolearn-docs-dice-group.netlify.app/usage/01_introduction)
 [![Python](https://img.shields.io/badge/python-3.10.13+-4584b6)](https://www.python.org/downloads/release/python-31013/)
  
 
 ![Ontolearn](docs/_static/images/Ontolearn_logo.png)
 
-# Ontolearn: Learning OWL Class Expression
+# Ontolearn: Learning OWL Class Expressions
 
 *Ontolearn* is an open-source software library for learning owl class expressions at large scale.
 

examples/retrieval_eval.py

@@ -46,26 +46,38 @@ def execute(args):
     ###################################################################
     # GENERATE ALCQ CONCEPTS TO EVALUATE RETRIEVAL PERFORMANCES
     # (3) R: Extract object properties.
-    object_properties = {i for i in symbolic_kb.get_object_properties()}
+    object_properties = sorted({i for i in symbolic_kb.get_object_properties()})
+
     # (3.1) Subsample if required.
     if args.ratio_sample_object_prob:
         object_properties = {i for i in random.sample(population=list(object_properties),
-                                                      k=max(1, int(len(object_properties) * args.ratio_sample_nc)))}
+                                                      k=max(1, int(len(object_properties) * args.ratio_sample_object_prob)))}
+
+    object_properties = set(object_properties)    
+
     # (4) R⁻: Inverse of object properties.
     object_properties_inverse = {i.get_inverse_property() for i in object_properties}
+
     # (5) R*: R UNION R⁻.
     object_properties_and_inverse = object_properties.union(object_properties_inverse)
     # (6) NC: Named owl concepts.
-    nc = {i for i in symbolic_kb.get_concepts()}
+    nc = sorted({i for i in symbolic_kb.get_concepts()})
+
+
+
+
     if args.ratio_sample_nc:
         # (6.1) Subsample if required.
         nc = {i for i in random.sample(population=list(nc), k=max(1, int(len(nc) * args.ratio_sample_nc)))}
+
+    nc = set(nc) # return to a set
     # (7) NC⁻: Complement of NC.
     nnc = {i.get_object_complement_of() for i in nc}
     # (8) UNNC: NC UNION NC⁻.
     unnc = nc.union(nnc)
+
     # (9) Retrieve 10 random Nominals.
-    nominals = set(random.sample(symbolic_kb.all_individuals_set(), 10))
+    nominals = set(random.sample(symbolic_kb.all_individuals_set(), 3))
     # (10) All Combinations of 3 for Nominals.
     nominal_combinations = set(
         OWLObjectOneOf(combination)
@@ -130,15 +142,17 @@ def concept_retrieval(retriever_func, c) -> Tuple[Set[str], float]:
     # () Converted to list so that the progress bar works.
     concepts = list(
         chain(
-            nc, unions, intersections, nnc, unnc, unions_unnc, intersections_unnc,
+            nc, unions, intersections, nnc, unions_unnc, intersections_unnc,
             exist_unnc, for_all_unnc,
             min_cardinality_unnc_1, min_cardinality_unnc_2, min_cardinality_unnc_3,
             max_cardinality_unnc_1, max_cardinality_unnc_2, max_cardinality_unnc_3,
             exist_nominals,
         )
     )
     # () Shuffled the data so that the progress bar is not influenced by the order of concepts.
+
     random.shuffle(concepts)
+
     # () Iterate over single OWL Class Expressions in ALCQIHO
     for expression in (tqdm_bar := tqdm(concepts, position=0, leave=True)):
         retrieval_y: Set[str]
@@ -159,6 +173,7 @@ def concept_retrieval(retriever_func, c) -> Tuple[Set[str], float]:
                 "Jaccard Similarity": jaccard_sim,
                 "F1": f1_sim,
                 "Runtime Benefits": runtime_y - runtime_neural_y,
+                "Runtime Neural": runtime_neural_y,
                 "Symbolic_Retrieval": retrieval_y,
                 "Symbolic_Retrieval_Neural": retrieval_neural_y,
             }
@@ -169,7 +184,7 @@ def concept_retrieval(retriever_func, c) -> Tuple[Set[str], float]:
         )
     # () Read the data into pandas dataframe
     df = pd.DataFrame(data)
-    assert df["Jaccard Similarity"].mean() == 1.0
+    assert df["Jaccard Similarity"].mean() >= args.min_jaccard_similarity
     # () Save the experimental results into csv file.
     df.to_csv(args.path_report)
     del df
@@ -191,13 +206,14 @@ def concept_retrieval(retriever_func, c) -> Tuple[Set[str], float]:
 
 def get_default_arguments():
     parser = ArgumentParser()
-    parser.add_argument("--path_kg", type=str, default="KGs/Family/family-benchmark_rich_background.owl")
+    parser.add_argument("--path_kg", type=str, default="KGs/Family/father.owl")
     parser.add_argument("--path_kge_model", type=str, default=None)
     parser.add_argument("--endpoint_triple_store", type=str, default=None)
-    parser.add_argument("--gamma", type=float, default=0.8)
+    parser.add_argument("--gamma", type=float, default=0.9)
     parser.add_argument("--seed", type=int, default=1)
-    parser.add_argument("--ratio_sample_nc", type=float, default=None, help="To sample OWL Classes.")
-    parser.add_argument("--ratio_sample_object_prob", type=float, default=None, help="To sample OWL Object Properties.")
+    parser.add_argument("--ratio_sample_nc", type=float, default=0.2, help="To sample OWL Classes.")
+    parser.add_argument("--ratio_sample_object_prob", type=float, default=0.1, help="To sample OWL Object Properties.")
+    parser.add_argument("--min_jaccard_similarity", type=float, default=0.0, help="Minimum Jaccard similarity to be achieve by the reasoner")
     # H is obtained if the forward chain is applied on KG.
     parser.add_argument("--path_report", type=str, default="ALCQHI_Retrieval_Results.csv")
     return parser.parse_args()