epsilon-compositionality

commit 4c5cc9e300012ca3bdcb175a17b9791985191479
parent cfa345f843fb8952bdac905f21e9e9d3deff0edb
Author: Sondre Wold <sondrewold1@gmail.com>
Date:   Fri, 13 Jun 2025 14:51:15 +0200

more filtering

Diffstat:
M
main.py
 | 
16
++++++++++------

1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/main.py b/main.py
@@ -22,11 +22,12 @@ class Point(NamedTuple):
 
 
 class Experiment:
-    def __init__(self: Experiment, model: str, dataset: dict[str, str], ngram: int, grams_are_tokens: bool, minimal_occurences: int) -> None:
+    def __init__(self: Experiment, model: str, dataset: dict[str, str], ngram: int, grams_are_tokens: bool, minimal_occurences: int, maximum_occurences: int) -> None:
         self.ngram = ngram
         self.dataset_name = dataset["name"]
         self.grams_are_tokens = grams_are_tokens
         self.minimal_occurences = minimal_occurences
+        self.maximum_occurences = maximum_occurences
         self.dataset = datasets.load_dataset(**dataset)
         self.model = transformers.AutoModel.from_pretrained(model).to(device)
         self.tokenizer = transformers.AutoTokenizer.from_pretrained(model)
@@ -55,7 +56,7 @@ class Experiment:
                     counters[key] += value
         return collections.defaultdict(collections.Counter, {
                 key: value for key, value in counters.items()
-                if sum(value.values()) >= self.minimal_occurences and len(value) >= 2
+                if sum(value.values()) >= self.minimal_occurences and sum(value.values()) <= self.maximum_occurences and len(value) >= 2
             })
 
     def embed(self: Experiment, text: str) -> torch.Tensor:
@@ -80,9 +81,11 @@ class Experiment:
                     cfirst = self.join_tokens([first, context])
                     for second, second_count in counter.items():
                         csecond = self.join_tokens([second, context])
+                        if cfirst == csecond:
+                            pass
                         yield Point(word_similarity=self.similarity(first, second),
                                     contextualized_similarity=self.similarity(cfirst, csecond),
-                                    weight=first_count/len(counter))
+                                    weight=first_count/sum(counter.values()))
 
     def __call__(self: Experiment):
         points: list[Point] = list(self.all_similarities())
@@ -103,8 +106,9 @@ if __name__ == "__main__":
     Experiment(
         model="bert-base-uncased",
         dataset={"path":"Salesforce/wikitext", "name":"wikitext-2-v1", "split":"train"},
-        #dataset={"path":"HuggingFaceTB/smollm-corpus", "name": "fineweb-edu-dedup", "split":"train"},
-        ngram=3,
+        #dataset={"path":"sedthh/gutenberg_english", "name": None, "split":"train"},
+        ngram=4,
         grams_are_tokens=False,
-        minimal_occurences=20,
+        minimal_occurences=10,
+        maximum_occurences=20,
     )()