From d5aab9a0a49cbc1a216c996532d3bbabe49ac47f Mon Sep 17 00:00:00 2001 From: Ryan Calvin Barron Date: Fri, 19 Sep 2025 11:40:12 -0600 Subject: [PATCH 01/13] spacey Ner block, hnmfk block error stack trace, termite package --- TELF/applications/Termite/VectorInjector.py | 30 + TELF/applications/Termite/__init__.py | 5 + .../Termite/embedding_store/__init__.py | 19 + .../Termite/embedding_store/base.py | 36 + .../Termite/embedding_store/milvus_store.py | 60 ++ .../embedding_store/opensearch_store.py | 164 +++++ .../applications/Termite/entities/__init__.py | 1 + .../Termite/entities/return_entities.py | 40 ++ .../Termite/neo4j_termite/DataInjector.py | 678 ++++++++++++++++++ .../Termite/neo4j_termite/__init__.py | 1 + .../Termite/neo4j_termite/constants.py | 108 +++ TELF/applications/Termite/termite.py | 317 ++++++++ TELF/applications/__init__.py | 6 + TELF/factorization/HNMFk.py | 132 +++- TELF/pipeline/__init__.py | 2 + TELF/pipeline/block_manager.py | 2 +- TELF/pipeline/blocks/__init__.py | 3 +- TELF/pipeline/blocks/sbatch_block.py | 131 +++- TELF/pipeline/blocks/semantic_hnmfk_block.py | 22 +- TELF/pipeline/blocks/spacey_NER_block.py | 213 ++++++ .../spacey_ner_block_example.ipynb | 134 ++++ examples/Lynx/README.md | 18 +- examples/Termite/01-Termite.ipynb | 597 +++++++++++++++ .../Termite/docker-compose-opensearch.yml | 86 +++ examples/Termite/docker-compose.yml | 123 ++++ post_install.py | 240 +++++-- pyproject.toml | 14 +- 27 files changed, 3067 insertions(+), 115 deletions(-) create mode 100644 TELF/applications/Termite/VectorInjector.py create mode 100644 TELF/applications/Termite/__init__.py create mode 100644 TELF/applications/Termite/embedding_store/__init__.py create mode 100644 TELF/applications/Termite/embedding_store/base.py create mode 100644 TELF/applications/Termite/embedding_store/milvus_store.py create mode 100644 TELF/applications/Termite/embedding_store/opensearch_store.py create mode 100755 TELF/applications/Termite/entities/__init__.py create mode 100755 TELF/applications/Termite/entities/return_entities.py create mode 100755 TELF/applications/Termite/neo4j_termite/DataInjector.py create mode 100755 TELF/applications/Termite/neo4j_termite/__init__.py create mode 100755 TELF/applications/Termite/neo4j_termite/constants.py create mode 100755 TELF/applications/Termite/termite.py create mode 100644 TELF/pipeline/blocks/spacey_NER_block.py create mode 100644 examples/Full TELF Pipeline/single_block_examples/spacey_ner_block_example.ipynb create mode 100755 examples/Termite/01-Termite.ipynb create mode 100644 examples/Termite/docker-compose-opensearch.yml create mode 100644 examples/Termite/docker-compose.yml diff --git a/TELF/applications/Termite/VectorInjector.py b/TELF/applications/Termite/VectorInjector.py new file mode 100644 index 00000000..8675a612 --- /dev/null +++ b/TELF/applications/Termite/VectorInjector.py @@ -0,0 +1,30 @@ +# Termite/VectorInjector.py +import torch +from transformers import AutoTokenizer, AutoModel +from typing import Iterable, List + +class Vectorizer: + """Computes embeddings; does NOT talk to any DB.""" + + def __init__(self, model_name: str = "malteos/scincl", device: str = None, max_length: int = 512): + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + self.model = AutoModel.from_pretrained(model_name) + self.model.eval().to(self.device) + self.max_length = max_length + + @torch.inference_mode() + def encode(self, texts: Iterable[str]) -> List[List[float]]: + outs = [] + for t in texts: + tokens = self.tokenizer( + t if isinstance(t, str) else "", + padding=True, + truncation=True, + max_length=self.max_length, + return_tensors="pt", + ).to(self.device) + hidden = self.model(**tokens).last_hidden_state # [1, L, H] + emb = hidden.mean(dim=1).squeeze(0) # mean pool + outs.append(emb.detach().cpu().tolist()) + return outs diff --git a/TELF/applications/Termite/__init__.py b/TELF/applications/Termite/__init__.py new file mode 100644 index 00000000..32e365f6 --- /dev/null +++ b/TELF/applications/Termite/__init__.py @@ -0,0 +1,5 @@ +# Keep this minimal: just re-export the public Termite class + +from .termite import Termite +from .neo4j_termite.constants import * +__all__ = ["Termite"] diff --git a/TELF/applications/Termite/embedding_store/__init__.py b/TELF/applications/Termite/embedding_store/__init__.py new file mode 100644 index 00000000..6493fb69 --- /dev/null +++ b/TELF/applications/Termite/embedding_store/__init__.py @@ -0,0 +1,19 @@ +# Termite/embedding_store/__init__.py +import os +from .base import EmbeddingStore +from .opensearch_store import OpenSearchStore +from .milvus_store import MilvusStore + +def make_store() -> EmbeddingStore: + backend = os.getenv("EMBEDDING_STORE", "opensearch").lower() + if backend == "milvus": + uri = os.getenv("MILVUS_URI", "http://localhost:19530") + return MilvusStore(uri=uri) + # default: OpenSearch + host = os.getenv("OS_HOST", "localhost") + port = int(os.getenv("OS_PORT", "9200")) + use_ssl = os.getenv("OS_USE_SSL", "false").lower() == "true" + username = os.getenv("OS_USERNAME") + password = os.getenv("OS_PASSWORD") + return OpenSearchStore(host=host, port=port, use_ssl=use_ssl, + username=username, password=password) \ No newline at end of file diff --git a/TELF/applications/Termite/embedding_store/base.py b/TELF/applications/Termite/embedding_store/base.py new file mode 100644 index 00000000..22823b80 --- /dev/null +++ b/TELF/applications/Termite/embedding_store/base.py @@ -0,0 +1,36 @@ +# Termite/embedding_store/base.py +from abc import ABC, abstractmethod +from typing import Iterable, List, Mapping, Optional, Tuple + +class EmbeddingStore(ABC): + """Abstract vector store interface.""" + + @abstractmethod + def ensure_index( + self, + index: str, + dim: int, + metric: str = "cosine", + **kwargs, + ) -> None: + """Create the index/collection if missing.""" + + @abstractmethod + def upsert( + self, + index: str, + ids: List[str], + vectors: List[List[float]], + payloads: Optional[List[Mapping]] = None, + ) -> None: + """Insert or update vectors with optional metadata.""" + + @abstractmethod + def search( + self, + index: str, + query: List[float], + k: int = 5, + **kwargs, + ) -> List[Tuple[str, float, Mapping]]: + """Return [(id, score, payload), ...].""" diff --git a/TELF/applications/Termite/embedding_store/milvus_store.py b/TELF/applications/Termite/embedding_store/milvus_store.py new file mode 100644 index 00000000..b133d1b4 --- /dev/null +++ b/TELF/applications/Termite/embedding_store/milvus_store.py @@ -0,0 +1,60 @@ +# Termite/embedding_store/milvus_store.py +from typing import List, Mapping, Optional, Tuple +from .base import EmbeddingStore + +from pymilvus import MilvusClient, DataType, FieldSchema, CollectionSchema, Collection +import numpy as np + +_METRIC_MAP = {"cosine": "COSINE", "l2": "L2", "dot": "IP"} + +class MilvusStore(EmbeddingStore): + def __init__(self, uri: str = "http://localhost:19530"): + self.client = MilvusClient(uri=uri) + + def ensure_index(self, index: str, dim: int, metric: str = "cosine", **kwargs) -> None: + if self.client.has_collection(index): + return + fields = [ + FieldSchema(name="id", dtype=DataType.VARCHAR, is_primary=True, max_length=128, auto_id=False), + FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=dim), + FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=4096), + FieldSchema(name="metadata", dtype=DataType.JSON), + ] + schema = CollectionSchema(fields=fields, description="Termite vectors") + self.client.create_collection(collection_name=index, schema=schema, consistency_level="Strong") + self.client.create_index( + collection_name=index, + field_name="embedding", + index_params={"index_type": "HNSW", "metric_type": _METRIC_MAP.get(metric.lower(), "COSINE"), "params": {"M": 16, "efConstruction": 200}}, + ) + + def upsert( + self, + index: str, + ids: List[str], + vectors: List[List[float]], + payloads: Optional[List[Mapping]] = None, + ) -> None: + rows = [] + for i, vid in enumerate(ids): + row = {"id": vid, "embedding": vectors[i]} + if payloads and i < len(payloads) and payloads[i] is not None: + # expect payload to contain "text" and/or "metadata" + row.update(payloads[i]) + rows.append(row) + self.client.insert(collection_name=index, data=rows) + + def search(self, index: str, query: List[float], k: int = 5, **kwargs) -> List[Tuple[str, float, Mapping]]: + res = self.client.search( + collection_name=index, + data=[query], + filter=kwargs.get("filter"), + limit=k, + output_fields=["id", "text", "metadata"], + search_params={"metric_type": "COSINE", "params": {"ef": kwargs.get("ef", 128)}}, + ) + hits = res[0] + out = [] + for h in hits: + out.append((h["entity"]["id"], float(h["distance"]), {"text": h["entity"].get("text"), "metadata": h["entity"].get("metadata")})) + return out diff --git a/TELF/applications/Termite/embedding_store/opensearch_store.py b/TELF/applications/Termite/embedding_store/opensearch_store.py new file mode 100644 index 00000000..7fc8371e --- /dev/null +++ b/TELF/applications/Termite/embedding_store/opensearch_store.py @@ -0,0 +1,164 @@ +# Termite/embedding_store/opensearch_store.py +from typing import List, Mapping, Optional, Tuple, Any +from .base import EmbeddingStore +from opensearchpy import OpenSearch, helpers +import os + +def _normalize_vector(vec: Any) -> List[float]: + """Accept list/tuple/np.ndarray or a string like '[0.1, 0.2]' → list[float].""" + if isinstance(vec, str): + import json, ast + try: + vec = json.loads(vec) + except Exception: + vec = ast.literal_eval(vec) + + try: + import numpy as np + if isinstance(vec, np.ndarray): + vec = vec.tolist() + except Exception: + pass + + if not isinstance(vec, list): + vec = list(vec) + + out = [] + for x in vec: + if x is None: + raise TypeError("Embedding contains None; all values must be floats.") + out.append(float(x)) + if not out: + raise ValueError("Embedding vector is empty.") + return out + +SPACE_MAP = {"cosine": "cosinesimil", "l2": "l2", "dot": "innerproduct"} + +class OpenSearchStore(EmbeddingStore): + def __init__( + self, + host: str = None, + port: int = None, + use_ssl: bool = None, + username: Optional[str] = None, + password: Optional[str] = None, + timeout: int = 60, + ): + # Allow env fallback (useful when Termite sets OS_* env vars) + host = host or os.environ.get("OS_HOST", "localhost") + port = int(port if port is not None else os.environ.get("OS_PORT", "9200")) + use_ssl = bool(str(use_ssl if use_ssl is not None else os.environ.get("OS_USE_SSL", "false")).lower() == "true") + + kwargs = { + "hosts": [{"host": host, "port": port}], + "use_ssl": use_ssl, + "verify_certs": False, # fine for dev + "http_compress": True, + "timeout": timeout, + } + if username and password: + kwargs["http_auth"] = (username, password) + self.client = OpenSearch(**kwargs) + + # ---------- Index management ---------- + def ensure_index( + self, + index: str, + dim: int, + metric: str = "cosine", + ef_search: Optional[int] = 128, # OS 2.13: set at index level (per-query requires >=2.16) + shards: int = 1, + replicas: int = 0, + engine: str = "nmslib", + **kwargs + ) -> None: + if self.client.indices.exists(index=index): + return + + space = SPACE_MAP.get(str(metric).lower(), "cosinesimil") + settings = { + "index": { + "knn": True, + "number_of_shards": int(shards), + "number_of_replicas": int(replicas), + } + } + if ef_search is not None: + # OpenSearch 2.13: tune HNSW candidate breadth here + settings["index"]["knn.algo_param.ef_search"] = int(ef_search) + + body = { + "settings": settings, + "mappings": { + "properties": { + "id": {"type": "keyword"}, + "text": {"type": "text"}, + "metadata": {"type": "object"}, + "embedding": { + "type": "knn_vector", + "dimension": int(dim), + "method": { + "name": "hnsw", + "engine": engine, + "space_type": space, + }, + }, + } + }, + } + self.client.indices.create(index=index, body=body) + + # ---------- Write ---------- + def upsert(self, index, ids, vectors, payloads=None, refresh: bool = True) -> None: + if len(ids) != len(vectors): + raise ValueError("ids and vectors must have the same length.") + actions = [] + for i, vid in enumerate(ids): + src = { + "id": str(vid), + "embedding": _normalize_vector(vectors[i]), + } + if payloads is not None and i < len(payloads) and payloads[i] is not None: + # merge user payloads (e.g., {"text": "...", "metadata": {...}}) + src.update(payloads[i]) + actions.append({"_op_type": "index", "_index": index, "_id": str(vid), "_source": src}) + helpers.bulk(self.client, actions) + if refresh: + self.client.indices.refresh(index=index) + + # ---------- Read ---------- + def search( + self, + index: str, + query, + k: int = 5, + source_fields: Optional[Any] = None, + ) -> List[Tuple[str, float, Mapping]]: + qvec = _normalize_vector(query) + + # _source handling: list or comma-sep string + if source_fields is None: + source_fields = ["id", "text", "metadata"] + elif isinstance(source_fields, str): + source_fields = [s.strip() for s in source_fields.split(",") if s.strip()] + elif not isinstance(source_fields, list): + raise TypeError(f"_source must be list[str] or comma-separated str; got {type(source_fields).__name__}") + + body = { + "size": int(k), + "query": { + "knn": { + # OpenSearch-native syntax for 2.13: + "embedding": {"vector": qvec, "k": int(k)} + } + }, + "_source": source_fields, + } + + resp = self.client.search(index=index, body=body) + hits = resp.get("hits", {}).get("hits", []) + out: List[Tuple[str, float, Mapping]] = [] + for h in hits: + src = h.get("_source", {}) or {} + out.append((h.get("_id", src.get("id")), float(h.get("_score", 0.0)), src)) + return out diff --git a/TELF/applications/Termite/entities/__init__.py b/TELF/applications/Termite/entities/__init__.py new file mode 100755 index 00000000..32392215 --- /dev/null +++ b/TELF/applications/Termite/entities/__init__.py @@ -0,0 +1 @@ +from .return_entities import * \ No newline at end of file diff --git a/TELF/applications/Termite/entities/return_entities.py b/TELF/applications/Termite/entities/return_entities.py new file mode 100755 index 00000000..2e73cc6f --- /dev/null +++ b/TELF/applications/Termite/entities/return_entities.py @@ -0,0 +1,40 @@ +class ReturnEntities: + def __init__(self): + self.all_returns = [] + + def add_ent(self, entity, attributes=None): + """ + Adds an entity to the list of returns. + + Parameters: + ----------- + entity : any + The entity to be added. + attributes : list of tuples, optional + The attributes of the entity. The default is None. + + Returns: + -------- + None + + """ + # Create a dictionary representing the entity and its attributes + entity_dict = { + "ENTITY": entity, + "ATTRIBUTES": attributes if attributes is not None else [] + } + # Append the entity dictionary to the list of returns + return self.all_returns.append(entity_dict) + + + def returns(self): + """ + Returns the list of entities. + + Returns: + -------- + list + A list of dictionaries, where each dictionary contains an entity and its attributes. + """ + # Return the list of entities + return self.all_returns diff --git a/TELF/applications/Termite/neo4j_termite/DataInjector.py b/TELF/applications/Termite/neo4j_termite/DataInjector.py new file mode 100755 index 00000000..e0286765 --- /dev/null +++ b/TELF/applications/Termite/neo4j_termite/DataInjector.py @@ -0,0 +1,678 @@ +from .constants import * +import numpy as np, pandas as pd, ast +from neo4j import GraphDatabase +from copy import deepcopy +from tqdm import tqdm +import math +import json +import csv +from concurrent.futures import ThreadPoolExecutor, as_completed + +class InjectorNeo4j: + def __init__(self, + kg_credentials=None, + verbose=False): + """ + Termite, Knowledge graph builder tool. + + Parameters + ---------- + verbose : bool, optional + Verbosity flag. The default is False. + kg_credentials : tuple[str, tuple[str,str]] + first string is url of graph + second tuple is auth, user and password + + Returns + ------- + None. + """ + self.kg_credentials = kg_credentials + self.verbose = verbose + + def setGraphCredentials(self,credentials): + """ + Set the knowledge graph credentials. + + Parameters: + ----------- + credentials : tuple[str, tuple[str,str]] + first string is url of graph + second tuple is auth, user and password + + Returns: + -------- + None + """ + self.kg_credentials = credentials + + def getGraphCredentials(self): + """ + set the knowledge graph credentials + + Parameters: + ----------- + None + + Returns: + -------- + tuple[str, tuple[str,str]] + first string is url of graph + second tuple is auth, user and password + """ + return self.kg_credentials + + + def add_triple(self, + data_container, # Dictionary containing data + head=np.nan, # Head node value + head_type=np.nan, # Head node type + head_attributes=np.nan, # Head node attributes + tail=np.nan, # Tail node value + tail_type=np.nan, # Tail node type + tail_attributes=np.nan, # Tail node attributes + relation=np.nan, # Relation between head and tail + weight=np.nan): # Weight of the relation + """ + Add a triple to the data_container dictionary. + + Parameters: + ----------- + data_container : dict + Dictionary containing data. + head : object, optional + Head node value. The default is np.nan. + head_type : object, optional + Head node type. The default is np.nan. + head_attributes : object, optional + Head node attributes. The default is np.nan. + tail : object, optional + Tail node value. The default is np.nan. + tail_type : object, optional + Tail node type. The default is np.nan. + tail_attributes : object, optional + Tail node attributes. The default is np.nan. + relation : object, optional + Relation between head and tail. The default is np.nan. + weight : object, optional + Weight of the relation. The default is np.nan. + """ + # Append data to data_container + data_container[H].append(head) + data_container[HT].append(head_type) + data_container[HA].append(head_attributes) + data_container[TT].append(tail_type) + data_container[T].append(tail) + data_container[R].append(relation) + data_container[W].append(weight) + data_container[TA].append(tail_attributes) + + def make_unique_constrains(self,column_triplet_map= None, verbose=False, additional_uniques=None): + if not column_triplet_map: + raise ValueError("Need a map for contraints") + + entities = column_triplet_map.get("ENTITIES") + if not entities: + raise ValueError("Need entities in the map") + + if verbose: + print(entities) + + URI, AUTH = self.getGraphCredentials() + with GraphDatabase.driver(URI, auth=AUTH) as driver: + for entity in entities: + make_unique = entity.get(MAKE_ID_UNIQUE) + if make_unique: + Head_type = entity.get(ET) + unique_contraint = f"CREATE CONSTRAINT {Head_type}_id_unique FOR (n:{Head_type}) REQUIRE n.id IS UNIQUE" + try: + + driver.execute_query(unique_contraint, + database_="neo4j", + ) + except Exception as e: + print(f"Failed to create unique constraint on: \n\t{unique_contraint} \n\t {e}") + + def from_csv_to_triplets(self, + csv_path, + save_path, + column_triplet_map=None): + """ + Builds a datafile that maps the raw csv into a head-relation-tail csv + + Parameters: + ----------- + csv_path : str + path to raw data + save_path : str + path to save mapped data + column_triplet_map : dict + Has entities and relations as keys and has a form that is of + the following where the subkeys are defined in the constants file : + {'ENTITIES':[{ ET:TYPE, FROM_COL: COL },], + 'RELATIONS':[{ HT:TYPE, R:TYPE, TT:TYPE, + EXTRACT_H:get_H, EXTRACT_T: get_T },]} + + Returns: + -------- + None + """ + verbose = self.verbose + df = pd.read_csv(csv_path) + + docs = {H:[], R:[], T:[], W:[], HT:[], TT:[], HA:[], TA:[]} + + for index, data in tqdm(df.iterrows(), total = len(df)): + row_entities = {} + # EXTRACT ENTITIES + for entity_map in column_triplet_map['ENTITIES']: + + # Added to unique contraints can be added in the map without prcessing as a column operator + # These entities are expected to have a function to extract them in the relations + if FROM_COL not in entity_map: + continue + + extraction_function = entity_map.get(EXTRACT_ENTITY) + if verbose: + print(entity_map) + + # If there is an extraction function, assume the function will also handle entity attributes + if extraction_function: + args = entity_map.get(ARGS) + if args: + if 'col' in args: # get the column value, remove col name from data + args['data'] = data[args['col']] + del args['col'] + entity = extraction_function(args) + else: + entity = extraction_function() + + + else: + entity = deepcopy(RETURN_TYPE) + if verbose: + print(f"entity_map[FROM_COL] ={entity_map[FROM_COL]}" ) + + if entity_map[FROM_COL] == ROW_INDEX: + entity[ENTITY] = int(index) + if verbose: + print('entity_map[FROM_COL] == ROW_INDEX') + else: + entity[ENTITY] = data[entity_map[FROM_COL]] + + attribute_cols = entity_map.get(ATTR_COL) + if attribute_cols: + for attribute in attribute_cols: + attribute_function = attribute.get(RETREIVAL) + if attribute_function: + # TODO: implement + pass + else: + attr_value = data[attribute[FROM_COL]] + + if type(attr_value) == str or type(attr_value) == int or (type(attr_value) == float and not math.isnan(attr_value)): + if entity[ATTRIBUTES] == None: + entity[ATTRIBUTES] = [(attribute[ATTR_NAME], attr_value)] + else: + entity[ATTRIBUTES].append((attribute[ATTR_NAME], attr_value)) + # else: + # print("Skipped attribute") + + if verbose: + print(f"entity={entity} for entity_map[ET] ={entity_map[ET]}") + + + row_entities[entity_map[ET]] = entity + + if verbose: + print(f"row_entities={row_entities}") + + # EXTRACT RELATIONS + for relation_map in column_triplet_map['RELATIONS']: + triple_details = {H:None, + T:None, + HT: relation_map[HT], + TT: relation_map.get(TT), + HA:None, + TA:None, + W:None, + R: relation_map.get(R), + } + + head_extraction = relation_map.get(EXTRACT_H) + head_args = relation_map.get(ARGS_H, {}) + head_args['data'] = data + + tail_extraction = relation_map.get(EXTRACT_T) + tail_args = relation_map.get(ARGS_T, {}) + tail_args['data'] = data + + # The heads and tails are embedded inline of the data row, and both must be exctracted through the functions passed + if head_extraction and tail_extraction: + head_entities = head_extraction(head_args) # extraction call, pass head type, must return a list of RETURN_TYPE + tail_entities = tail_extraction(tail_args) # extraction call, pass tail type, must return a list of RETURN_TYPE + + pairing = relation_map.get(PAIRING) + if pairing == MANY_TO_MANY or not pairing: # default if no pairing specified + for i, head_entity in enumerate(head_entities): + triple_details[H] = head_entity[ENTITY] + triple_details[W] = head_entity[W] + triple_details[HA] = row_entities[triple_details[HT]][ATTRIBUTES] + + for tail_entity in tail_entities: + if i > 0: + triple_details[HA] = None + triple_details[T] = tail_entity[ENTITY] + triple_details[TA] = tail_entity[ATTRIBUTES] + triple_details[W] = tail_entity[W] + + self.add_triple(docs, **triple_details) + + else: # index pairing + heads_len = len(head_entities) + tails_len = len(tail_entities) + if verbose: + print(head_entities) + print(tail_entities) + print(f"heads_len ={heads_len}, tails_len={tails_len}") + assert heads_len == tails_len + for head_entity, tail_entity in zip(head_entities, tail_entities): + triple_details[H] = head_entity[ENTITY] + triple_details[HA] = head_entity[ATTRIBUTES] + triple_details[W] = head_entity[W] + + triple_details[T] = tail_entity[ENTITY] + triple_details[TA] = tail_entity[ATTRIBUTES] + triple_details[W] = tail_entity[W] + + if triple_details[H] not in [None, 'None'] and triple_details[T] not in [None, 'None']: + self.add_triple(docs, **triple_details) + + # The heads only are embedded inline of the data row, and must be exctracted through the function passed + elif head_extraction: + head_entities = head_extraction(head_args) # extraction call, must return a list of RETURN_TYPE + + if triple_details[TT]: + triple_details[T] = row_entities[triple_details[TT]][ENTITY] + triple_details[TA] = row_entities[triple_details[TT]][ATTRIBUTES] + + + for i, head_entity in enumerate(head_entities): + if i > 0 and triple_details[TT]: + triple_details[TA] = None + + triple_details[H] = head_entity[ENTITY] + triple_details[HA] = head_entity[ATTRIBUTES] + triple_details[W] = head_entity[W] + + if triple_details[H] not in [None, 'None'] and triple_details[T] not in [None, 'None']: + self.add_triple(docs, **triple_details) + + # The tails only are embedded inline of the data row, and must be exctracted through the function passed + elif tail_extraction: + tail_entities = tail_extraction(tail_args ) # extraction call, must return a list of RETURN_TYPE + triple_details[H] = row_entities[triple_details[HT]][ENTITY] # Head is the Row entity looked up through the head type in the details + triple_details[HA] = row_entities[triple_details[HT]][ATTRIBUTES] + + + # print(f'extracting tail for {triple_details[H] }') + + for i , tail_entity in enumerate(tail_entities): + if i > 0: + triple_details[HA] = None + + + triple_details[T] = tail_entity[ENTITY] + triple_details[W] = tail_entity[W] + triple_details[TA] = tail_entity[ATTRIBUTES] + + if triple_details[H] not in [None, 'None'] and triple_details[T] not in [None, 'None']: + # print('Adding triplet') + self.add_triple(docs, **triple_details) + + # The head and tail are both accessed directly through their column maps + else: + if verbose: + print(f"triple_details[HT] = {triple_details[HT]}") + triple_details[H] = row_entities[triple_details[HT]][ENTITY] # Head is the Row entity looked up through the head type in the details + triple_details[HA] = row_entities[triple_details[HT]][ATTRIBUTES] + if triple_details[TT]: + triple_details[T] = row_entities[triple_details[TT]][ENTITY] + triple_details[TA] = row_entities[triple_details[TT]][ATTRIBUTES] + triple_details[W] = row_entities[triple_details[TT]][W] + if verbose: + print(f"triple_details = {triple_details}") + + # if triple_details[H] not in [None, 'None'] and triple_details[T] not in [None, 'None']: + self.add_triple(docs, **triple_details) + + KG_df = pd.DataFrame.from_dict(docs) + KG_df.to_csv(save_path, index=False) + + + def make_attribute_string(self, + attributes, + node_id): + """ + Attribute query contructor for nodes to set the attribuets in the KG + + Parameters: + ----------- + attributes : list + list of tuples containing the attribute name/id and its value + node_id: + which node to assign the attribute + + Returns: + -------- + str + part of a query containing attributes to be appended to a larger query + """ + query_part = '' + attributes = ast.literal_eval(attributes) + if len(attributes): + for attribute in attributes: + attribute_identifier = attribute[0] + attribute_value = attribute[1] + query_part += "SET "+node_id+"." + attribute_identifier + "='" + str(attribute_value).replace('"', '\\"').replace('\'', '\\\'').replace('\\\\\'', '\\\'') + "'" + return query_part + + + def make_triples(self, + driver, + node1, + node1_type, + relation, + node2, + node2_type, + weight, + head_attributes, + tail_attributes, + index = 0): + """ + Parses data into query format for Knowledge graph. Injects query in final step using the driver. + + Parameters: + ----------- + driver : KG driver + to operate the queries + node1 : str + The ID of node 1 + node1_type : str + The graph label for node 1 + relation : str + The edge's graph label + node2 : str + The ID of node + node2_type : str + The graph label for node 2 + weight : float + Weight for the edge + head_attributes : list + Attributes to assign to the head node + tail_attributes : list + Attributes to assign to the tail node + index : int + Current index of Head-Entity-Tail iteration. Helps identify issues in data + + Returns: + -------- + None, int + returns the negative index of queries with problems. No issue, nothing is returned. + """ + # MAKE SURE HEAD NOT NULL + for val in [node1, node1_type]: + if type(val) == float and math.isnan(val): + return -index + + # MAKE ID INT IF NOT STRING + try: + node1 = int(node1) + except: + pass + + # GENERATE HEAD AND ITS ATTRIBUTES + query = "MERGE (a:"+node1_type+" {id: $node1} )" + if type(head_attributes) == str: # not math.isnan(head_attributes): + query += self.make_attribute_string(head_attributes, node_id='a') + + # IF TAIL AND RELATION ARE NOT NULL< GENERATE THEM + skip_relation_tail = False + for val in [relation, node2, node2_type]: + if type(val) == float and math.isnan(val): + skip_relation_tail = True + + if not skip_relation_tail: + try: + node2 = int(node2) + except: + pass + + # GENERATE TAIL AND ITS ATTRIBUTES + query += "MERGE (b:"+node2_type+" {id: $node2} )" + if type(tail_attributes) == str: #not math.isnan(tail_attributes): + query += self.make_attribute_string(tail_attributes, node_id='b') + + # RELATION and optional WEIGHT + if math.isnan(weight): + query += "MERGE (a)-[:"+relation+" ]->(b)" + else: + query += "MERGE (a)-[:"+relation+" {weight: $weight} ]->(b)" + + + # Makes EDGES and NODES in NEO4J + driver.execute_query(query, + node1=node1, node2=node2, weight=weight, + database_="neo4j", + ) + + + def iterate_csv_triplets_into_graph(self, + triplets_path, + start_from =0, + args={}): + """ + Iterates parsed data to call the injection to graph function. + + Parameters: + ----------- + triplets_path : str + Path to the mapped data + args : dict + any additions to be made + + Returns: + -------- + list + Indicies of failed injections + """ + + df = pd.read_csv(triplets_path) + + if start_from: + start = int(len(df)* start_from) + df = df.iloc[start:] + + failed_indicies = [] + + URI, AUTH = self.getGraphCredentials() + with GraphDatabase.driver(URI, auth=AUTH) as driver: + for index, data in tqdm(df.iterrows(), total = len(df)): + + node1 = data[H] + node2 = data[T] + + node1_type = data[HT] + node2_type = data[TT] + + relation = data[R] + weight = data[W] + + head_attributes = data.get(HA, 'nan') + tail_attributes = data.get(TA, 'nan') + index_on_fail = self.make_triples(driver, node1, node1_type, relation, node2, node2_type, weight,head_attributes, tail_attributes, index) + if index_on_fail: + failed_indicies.append(index_on_fail) + + return failed_indicies + + def process_row(self, + data, + index, + driver): + """ + Process a row of data and make triples using the provided driver. + + :param data: The data dictionary containing information about nodes, relations, and attributes. + :param index: The index of the row being processed. + :param driver: The driver object for interacting with the database. + :return: The triples generated by the function call to make_triples. + """ + + node1 = data[H] + node2 = data[T] + + node1_type = data[HT] + node2_type = data[TT] + + relation = data[R] + weight = data[W] + + head_attributes = data.get(HA, 'nan') + tail_attributes = data.get(TA, 'nan') + + return self.make_triples(driver, node1, node1_type, relation, node2, node2_type, weight, head_attributes, tail_attributes, index) + + def update_database_multithreaded(self, + triplets_path, + start_from=0, + shuffle_rows=True): + """ + Iterates parsed data to call the injection to graph function. + + The code does not handle iterations with shuffle_rows=True and start_from != 0. + If shuffle_rows with start_from is needed, manually modify data to accomodate then repass the moddified path. + + Parameters: + ----------- + triplets_path : str + Path to the mapped data + start_from : int + Which row to start from + shuffle_rows : bool + Random shuffle of rows to prevent Neo4j from deadlocking on rows that have common nodes + + Returns: + -------- + list + Indicies of failed injections + """ + df = pd.read_csv(triplets_path) + + if shuffle_rows: + df = df.sample(frac=1).reset_index(drop=True) + df.to_csv(triplets_path) + + failed_indices = [] + + URI, AUTH = self.getGraphCredentials() + + if start_from: + start = int(len(df)* start_from) + print(f"skipping {start}") + df = df.iloc[start:] + + with GraphDatabase.driver(URI, auth=AUTH) as driver: + with ThreadPoolExecutor() as executor: + futures = [executor.submit(self.process_row, data, index, driver) for index, data in df.iterrows()] + # print(len(futures)) + for future in tqdm(as_completed(futures), total=len(futures)): + + index_on_fail = future.result() + if index_on_fail: + failed_indices.append(index_on_fail) + + return failed_indices + + + def write_csv(self, filename, data, headers): + """ + Writes data to a CSV file with specified headers. + + Parameters: + ----------- + filename : str + Path to the CSV file. + data : list of dict + Data to be written to the CSV file. + headers : list of str + Headers for the CSV file. + + Returns: + -------- + None + """ + + # Open the CSV file in 'w' mode with UTF-8 encoding + with open(filename, 'w', newline='', encoding='utf-8') as csvfile: + # Create a DictWriter object from the opened file + writer = csv.DictWriter(csvfile, fieldnames=headers) + + # Write the headers to the CSV file + writer.writeheader() + + # Iterate over the data and write each row to the CSV file + for row in data: + writer.writerow(row) + + def process_json_to_csv(self, + json_path): + """ + Process a JSON file containing nodes and relationships and write + them to CSV files. + + Parameters: + ----------- + json_path : str + Path to the JSON file. + + Returns: + -------- + None + """ + + # Lists to store node and relationship data + nodes_data = [] + relationships_data = [] + + # Open and iterate through the file line by line + with open(json_path, 'r', encoding='utf-8') as file: + for line in file: + try: + # Parse the line as JSON + obj = json.loads(line) + except json.JSONDecodeError: + continue # Skip lines that can't be parsed as JSON + + # If the object is a node, extract its data + if obj['type'] == 'node': + properties = obj.get('properties', {}) + properties[':ID'] = obj.get('id') + properties[':LABEL'] = ';'.join(obj.get('labels', [])) + nodes_data.append(properties) + # If the object is a relationship, extract its data + elif obj['type'] == 'relationship': + properties = obj.get('properties', {}) + properties[':START_ID'] = obj.get('start', {}).get('id') + properties[':END_ID'] = obj.get('end', {}).get('id') + properties[':TYPE'] = obj.get('label') + relationships_data.append(properties) + + # Define CSV headers dynamically based on collected keys + node_headers = list(set(key for row in nodes_data for key in row)) + relationship_headers = list(set(key for row in relationships_data for key in row)) + + # Write node data to a CSV file + self.write_csv('nodes.csv', nodes_data, node_headers) + # Write relationship data to a CSV file + self.write_csv('relationships.csv', relationships_data, relationship_headers) diff --git a/TELF/applications/Termite/neo4j_termite/__init__.py b/TELF/applications/Termite/neo4j_termite/__init__.py new file mode 100755 index 00000000..9737af31 --- /dev/null +++ b/TELF/applications/Termite/neo4j_termite/__init__.py @@ -0,0 +1 @@ +from .constants import * \ No newline at end of file diff --git a/TELF/applications/Termite/neo4j_termite/constants.py b/TELF/applications/Termite/neo4j_termite/constants.py new file mode 100755 index 00000000..b6ef7ee3 --- /dev/null +++ b/TELF/applications/Termite/neo4j_termite/constants.py @@ -0,0 +1,108 @@ +import numpy as np + +""" +General FIELDS +""" +MAKE_ID_UNIQUE = 'unique' + +""" +TRIPLET CSV FIELDS +""" +H = 'head' +HT = 'head_type' +T = 'tail' +TT = 'tail_type' +R = 'relation' +W = 'weight' +HA = 'head_attributes' +TA = 'tail_attributes' +ET = "entity_type" +ROW_INDEX = 'index' +FROM_COL = "from_column" +ATTR_COL = 'attribute_columns' +ATTR_FUNC = 'attribute_function' +ATTR_NAME = 'attribute_name' +ARGS = 'args' +RETREIVAL = 'retrival_operation' +ENTITY = 'entity' +ATTRIBUTES = 'attributes' +EXTRACT_H = 'head_extraction_function' +EXTRACT_T = 'tail_extraction_function' +EXTRACT_ENTITY = 'extract_entity' +PAIRING = "ordering_pairing" +HEAD_TO_MANY = 'head_to_many' +INDEX_PAIRING = 'preserve_index' +MANY_TO_MANY = 'many_to_many' +MANY_TO_TAIL = 'many_to_tail' +ARGS_H = 'head_arguments' +ARGS_T = 'tail_arguments' + + +EMPTY_VALUES = [np.nan, 'None', 'nan'] +RETURN_TYPE = {ENTITY:None, W: None, ATTRIBUTES:None} + +""" +HEADS AND TAILS / NODE TYPING +""" +YEAR_TYPE = 'Year' +TOPIC_TYPE = 'Topic_ID' +COUNTRY_TYPE = 'Country' +KEYWORD_TYPE = 'Keyword' +AUTHOR_ID_TYPE = 'Author_ID' +DOCUMENT_TYPE = 'Document_ID' +DOCUMENT_TYPE_SCOPUS = 'Document_ID_SCOPUS' +SME_WORDS_TYPE = 'SME_word_tag' +NAMED_ENTITY_TYPE = 'Named_Entity' +AFFILIATION_IDENTIFIER_TYPE = 'Affiliation_ID' +PUBLISHER = 'Publisher' +CATEGORY = 'Scopus_category' + +MATERIAL_TYPE = "Material" +AUX_MATERIAL_TYPE = "Auxillary_Material" + + +NER_LOCATION = "NER_location" +NER_PRODUCT = 'NER_product' +NER_ORGANIZATION = 'NER_organization' +NER_GEOPOLITICAL = 'NER_geopolitical_entity' +SME_KEYWORD = 'sme_keyword' +ACRONYM = 'acronym' + + + +""" +RELATION / EDGE TYPES +""" +DOCUMENT_CITES_RELATION = 'cites' +AUTHOR_DOCUMENT_RELATION = 'wrote' +DOCUMENT_CITED_RELATION = 'cited_by' +TOPIC_KEYWORD_RELATION = 'is_about' +DOCUMENT_YEAR_RELATION = 'written_in_year' +DOCUMENT_TOPIC_RELATION = 'is_part_of_topic' +DOCUMENT_AFFILITATION_RELATION = 'is_affiliated_with' +AUTHOR_AFFILITATION_RELATION = 'is_affiliated_with' +AFFILIATION_COUNTRY_RELATION = 'is_in_country' +DOCUMENT_SME_WORD_RELATION = 'has_sme_word' +DOCUMENT_PUBLISHER_RELATION = 'published_by' +DOCUMENT_CATEGORY_RELATION = 'is_in_category' + +DOCUMENT_MATERIAL_RELATION = 'mentions_material' + +MATERIAL_TOPIC_RELATION = 'in_topic' +DOCUMENT_MAIN_MATERIAL_RELATION = 'mentions_main_material' +DOCUMENT_AUX_MATERIAL_RELATION = 'mentions_aux_material' + +TOPIC_AUX_MATERIAL_RELATION = 'contains_aux_material' +DOCUMENT_SME_RELATION='mentions_sme_keyword' +DOCUMENT_LOCATION_RELATION='mentions_location' +DOCUMENT_PRODUCT_RELATION='mentions_product' +DOCUMENT_ORGANIZATION_RELATION='mentions_organization' +DOCUMENT_GEOPOLITICAL_RELATION='mentions_geopolitical_entity' +DOCUMENT_ACRONYM_RELATION='mentions_acronym' + +# WITH ['Topic_ID', 'Keyword', 'Document_ID', 'Document_ID_SCOPUS', 'Affiliation_ID', 'Country', +# 'Year', 'Author_ID', 'SME_word_tag', 'Publisher', 'Scopus_category' ] AS labels +# FOREACH (label IN labels | +# CREATE CONSTRAINT FOR (node:`$label`) REQUIRE node.neo4jImportId IS UNIQUE; +# ) + diff --git a/TELF/applications/Termite/termite.py b/TELF/applications/Termite/termite.py new file mode 100755 index 00000000..2ec30a44 --- /dev/null +++ b/TELF/applications/Termite/termite.py @@ -0,0 +1,317 @@ +# Termite/termite.py +from __future__ import annotations + +import os +from typing import Dict, List, Mapping, Optional, Tuple, Iterable, Any, Union + +from .neo4j_termite.constants import * +from .neo4j_termite.DataInjector import InjectorNeo4j + +# NEW: storage-agnostic vectorizer + store factory +# - Vectorizer computes embeddings only +# - make_store() returns an OpenSearch (default) or Milvus backend +from .VectorInjector import Vectorizer +from .embedding_store import make_store # returns EmbeddingStore + +import pandas as pd + + +class Termite: + """ + Termite, Knowledge graph builder tool. + + Now storage-agnostic for embeddings: + - Default backend: OpenSearch (k-NN) + - Optional backend: Milvus (set EMBEDDING_STORE=milvus or pass embedding_backend='milvus') + + Backward-compat params kept: + - vector_uri: used if backend == 'milvus'; ignored for OpenSearch unless you pass + embedding_backend_config={'host': ..., 'port': ...} + """ + + def __init__( + self, + kg_credentials: Optional[Tuple[str, Tuple[str, str]]] = None, + vector_uri: str = "http://localhost:19530", + db_nme: str = "default", + token: Optional[str] = None, + verbose: bool = False, + embedding_backend: Optional[str] = None, + embedding_backend_config: Optional[Mapping[str, Any]] = None, + model_name: str = "malteos/scincl", + use_gpu: bool = False, + ): + """ + Parameters + ---------- + kg_credentials : tuple[str, tuple[str,str]] + (url, (user, password)) for Neo4j + vector_uri : str + Back-compat. Used as Milvus URI when embedding_backend='milvus'. + db_nme : str + Not used by OpenSearch; kept for compatibility. + token : str | None + Reserved for future auth needs. + verbose : bool + Verbosity flag. + embedding_backend : {'opensearch','milvus'} | None + If None, falls back to ENV EMBEDDING_STORE or 'opensearch'. + embedding_backend_config : Mapping + Backend-specific knobs, e.g. + OpenSearch: {'host':'localhost','port':9200,'use_ssl':False} + Milvus: {'uri':'http://localhost:19530'} + model_name : str + HF model to compute embeddings. + use_gpu : bool + Try to use CUDA for embeddings. + """ + self.verbose = verbose + + # ---- Graph injector (unchanged) ---- + self.graph_injector = InjectorNeo4j(kg_credentials, verbose) + + # ---- Vectorizer (compute embeddings only) ---- + self.vectorizer = Vectorizer(model_name=model_name) + + # ---- Embedding store (pluggable) ---- + # precedence: explicit arg -> ENV -> default('opensearch') + backend = (embedding_backend or os.getenv("EMBEDDING_STORE", "opensearch")).lower() + cfg = dict(embedding_backend_config or {}) + + if backend == "milvus": + # honor legacy vector_uri if not provided in config + cfg.setdefault("uri", vector_uri) + os.environ.setdefault("EMBEDDING_STORE", "milvus") + else: + # default OpenSearch + os.environ.setdefault("EMBEDDING_STORE", "opensearch") + # allow simple overrides + cfg.setdefault("host", os.getenv("OS_HOST", "localhost")) + cfg.setdefault("port", int(os.getenv("OS_PORT", "9200"))) + cfg.setdefault("use_ssl", os.getenv("OS_USE_SSL", "false").lower() == "true") + + # make_store() reads EMBEDDING_STORE + common OS_* / MILVUS_* envs; + # we pass overrides via env for a simple, consolidated factory. + if backend == "milvus": + if "uri" in cfg: + os.environ["MILVUS_URI"] = str(cfg["uri"]) + else: + os.environ["OS_HOST"] = str(cfg["host"]) + os.environ["OS_PORT"] = str(cfg["port"]) + os.environ["OS_USE_SSL"] = "true" if cfg.get("use_ssl") else "false" + + self.store = make_store() + + # book-keeping (optional) + self._default_metric = "cosine" + self._db_name = db_nme + self._token = token + self._model_name = model_name + self._use_gpu = use_gpu + + # ------------------------- + # CSV → triplets (graph) + # ------------------------- + + def from_csv_to_triplets(self, csv_path: str, save_path: str, column_triplet_map: Optional[Mapping] = None): + """ + Builds a datafile that maps the raw csv into a head-relation-tail csv + """ + self.graph_injector.from_csv_to_triplets(csv_path, save_path, column_triplet_map) + + def make_unique_constrains(self, column_triplet_map: Optional[Mapping] = None, verbose: bool = False): + self.graph_injector.make_unique_constrains(column_triplet_map, verbose) + + def iterate_csv_triplets_into_graph(self, triplets_path: str, start_from: int = 0, args: Mapping = {}): + """ + Iterates parsed data to call the injection to graph function. + Returns list of failed indices. + """ + self.graph_injector.iterate_csv_triplets_into_graph(triplets_path, start_from, args) + + def update_database_multithreaded(self, triplets_path: str, start_from: int = 0, shuffle_rows: bool = True): + """ + Iterates parsed data to call the injection to graph function (multithreaded). + """ + self.graph_injector.update_database_multithreaded(triplets_path, start_from, shuffle_rows) + + # ------------------------- + # VECTOR STORE (generic) + # ------------------------- + + def make_vector_schema(self, collection_name: str = "slic", schema: Optional[Mapping] = None, dim: Optional[int] = None, metric: Optional[str] = None): + """ + Creates an index/collection in the selected vector backend. + + For OpenSearch, 'schema' is ignored; we derive a simple mapping with a single + knn_vector field named 'embedding', plus 'id', 'text', 'metadata'. + + Parameters + ---------- + collection_name : str + schema : dict | None + (Kept for compatibility; Milvus users can pass params if they want.) + dim : int | None + Dimension of the embedding. If None, we try to infer from 'schema', + otherwise you must supply it before inserting data. + metric : str | None + 'cosine' (default), 'l2', or 'dot' + """ + metric = (metric or self._default_metric).lower() + if dim is None: + # Try to infer from schema (Milvus-style) else refuse early + if isinstance(schema, dict) and "dim" in schema: + dim = int(schema["dim"]) + else: + raise ValueError("make_vector_schema requires 'dim' (embedding dimension).") + self.store.ensure_index(index=collection_name, dim=dim, metric=metric) + + def inject_vectors(self, collection_name: str, data: List[Mapping[str, Any]]): + """ + Insert / upsert vectors into the selected backend. + + Expected 'data' shape (per item): + { + 'id': str, + 'embedding': List[float], # length = dim + # optional user payload: + 'text': str, + 'metadata': Mapping[str, Any] + } + """ + ids: List[str] = [] + vecs: List[List[float]] = [] + payloads: List[Mapping[str, Any]] = [] + + for row in data: + if "id" not in row or "embedding" not in row: + raise ValueError("Each data row must contain 'id' and 'embedding'.") + ids.append(str(row["id"])) + vecs.append(list(row["embedding"])) + payloads.append({k: v for k, v in row.items() if k not in ("id", "embedding")}) + + self.store.upsert(index=collection_name, ids=ids, vectors=vecs, payloads=payloads) + + # ------------------------- + # HELPERS to shape data + # ------------------------- + + def df_to_data( + self, + embeddings: List[List[float]], + df_path: str, + columns_collection_map: Mapping[str, str], + id_col: Optional[str] = None, + text_col: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + Converts a CSV into a list of upsertable vector rows. + + Parameters + ---------- + embeddings : list[list[float]] + Embeddings in the same row-order as the CSV. + df_path : str + Path to CSV. + columns_collection_map : dict + Mapping of wanted CSV columns → payload keys (e.g., {"abstract":"text","paper_id":"id"}). + If 'id' or 'text' are not provided here, we will fallback to id_col/text_col args. + id_col, text_col : Optional[str] + Fallbacks for the identifier and text payload. + + Returns + ------- + List[dict] + Each dict at least has {'id', 'embedding'} plus any mapped payload fields. + """ + df = pd.read_csv(df_path) + if len(df) != len(embeddings): + raise ValueError(f"embeddings length ({len(embeddings)}) != rows in CSV ({len(df)})") + + # Determine id/text columns + mapped_id = next((src for src, dst in columns_collection_map.items() if dst == "id"), None) or id_col + mapped_text = next((src for src, dst in columns_collection_map.items() if dst == "text"), None) or text_col + + data: List[Dict[str, Any]] = [] + for i, row in df.iterrows(): + payload: Dict[str, Any] = {} + + # Map requested columns + for src, dst in columns_collection_map.items(): + if src in df.columns: + payload[dst] = row[src] + + # Ensure id/text presence if desired + rid = str(row[mapped_id]) if mapped_id and mapped_id in df.columns else str(i) + if "id" not in payload: + payload["id"] = rid + + if mapped_text and mapped_text in df.columns and "text" not in payload: + payload["text"] = row[mapped_text] + + data.append({"id": payload.pop("id"), "embedding": embeddings[i], **payload}) + + return data + + # ------------------------- + # EMBEDDING COMPUTATION + # ------------------------- + + def compute_embeddings( + self, + df: "pd.DataFrame", + model_name: str = "SCINCL", + use_gpu: bool = False, + text_column: Optional[str] = None, + ) -> Dict[Any, List[float]]: + """ + Computes embeddings for a pandas DataFrame. + + Assumes there is a textual column to embed. If `text_column` is None, + we try 'text', then the first object/string dtype column. + + Returns + ------- + dict: {row_index -> embedding_vector} + """ + # choose column + col = text_column + if col is None: + if "text" in df.columns: + col = "text" + else: + # pick first object-dtype column as a best-effort fallback + obj_cols = [c for c in df.columns if df[c].dtype == object] + if not obj_cols: + raise ValueError("No text column found. Provide 'text_column'.") + col = obj_cols[0] + + texts: List[str] = df[col].astype(str).tolist() + + # swap model if caller passed one + if model_name and model_name != self._model_name: + self.vectorizer = Vectorizer(model_name=model_name) + self._model_name = model_name + + # note: Vectorizer selects CUDA automatically if available; we keep the flag for API parity + embs = self.vectorizer.encode(texts) + + # map back to index + return {idx: embs[i] for i, idx in enumerate(df.index.tolist())} + + # ------------------------- + # OPTIONAL: convenience search + # ------------------------- + + def search_vectors( + self, + collection_name: str, + query_vec: List[float], + k: int = 5, + **kwargs, + ) -> List[Tuple[str, float, Mapping]]: + """ + Simple k-NN search wrapper to the active backend. + Returns a list of (id, score, payload). + """ + return self.store.search(index=collection_name, query=query_vec, k=k, **kwargs) diff --git a/TELF/applications/__init__.py b/TELF/applications/__init__.py index 9d88e0ec..764a6383 100644 --- a/TELF/applications/__init__.py +++ b/TELF/applications/__init__.py @@ -2,6 +2,7 @@ sys.path += ["Cheetah"] sys.path += ["Bunny"] sys.path += ["Penguin"] +sys.path += ["Termite"] # Cheetah @@ -16,3 +17,8 @@ # Penguin from .Penguin.penguin import Penguin + +# Termite +from .Termite.termite import Termite +from .Termite.neo4j_termite.constants import * + diff --git a/TELF/factorization/HNMFk.py b/TELF/factorization/HNMFk.py index 14d5040e..22efa6cc 100644 --- a/TELF/factorization/HNMFk.py +++ b/TELF/factorization/HNMFk.py @@ -609,21 +609,22 @@ def _process_node(self, Ks, # check if leaf node status based on number of samples # if (current_node.num_samples == 1): - current_node.leaf = True - current_node.exception = True - pickle_path = os.path.join(str(node_save_path), f'node_{current_node.node_name}.p') - pickle.dump(current_node, open(pickle_path, "wb")) - return {"name":node_name, "target_jobs":[], "node_save_path":pickle_path} - + return self._finalize_leaf( + current_node, node_save_path, + reason="single-sample cluster", + details={"num_samples": current_node.num_samples}, + mark_exception=True + ) # # Sample threshold check for leaf node determination # if self.sample_thresh > 0 and (current_node.num_samples <= self.sample_thresh): - current_node.leaf = True - pickle_path = os.path.join(f'{node_save_path}', f'node_{current_node.node_name}.p') - pickle.dump(current_node, open(pickle_path, "wb")) - return {"name":node_name, "target_jobs":[], "node_save_path":pickle_path} + return self._finalize_leaf( + current_node, node_save_path, + reason="below sample threshold", + details={"num_samples": current_node.num_samples, "sample_thresh": self.sample_thresh} + ) # # obtain the current X @@ -644,12 +645,13 @@ def _process_node(self, Ks, # Based on number of features or samples, no seperation possible # if min(curr_X.shape) <= 1: - current_node.leaf = True - current_node.exception = True - pickle_path = os.path.join(f'{node_save_path}', f'node_{current_node.node_name}.p') - pickle.dump(current_node, open(pickle_path, "wb")) - return {"name":node_name, "target_jobs":[], "node_save_path":pickle_path} - + return self._finalize_leaf( + current_node, node_save_path, + reason="matrix too small to factorize", + details={"X_shape": tuple(curr_X.shape)}, + mark_exception=True + ) + # # prepare the current nmfk parameters # @@ -665,11 +667,12 @@ def _process_node(self, Ks, # Ks = self._adjust_curr_Ks(curr_X.shape, Ks) if len(Ks) == 0 or (len(Ks) == 1 and Ks[0] < 2): - current_node.leaf = True - current_node.exception = True - pickle_path = os.path.join(str(node_save_path), f'node_{current_node.node_name}.p') - pickle.dump(current_node, open(pickle_path, "wb")) - return {"name":node_name, "target_jobs":[], "node_save_path":pickle_path} + return self._finalize_leaf( + current_node, node_save_path, + reason="no valid K values for further split", + details={"Ks": list(Ks), "X_shape": tuple(curr_X.shape)}, + mark_exception=True + ) # # apply nmfk @@ -681,11 +684,12 @@ def _process_node(self, Ks, # Check if decomposition was not possible # if results is None: - current_node.leaf = True - current_node.exception = True - pickle_path = os.path.join(str(node_save_path), f'node_{current_node.node_name}.p') - pickle.dump(current_node, open(pickle_path, "wb")) - return {"name":node_name, "target_jobs":[], "node_save_path":pickle_path} + return self._finalize_leaf( + current_node, node_save_path, + reason="NMFk failed to decompose", + details={"node_depth": current_node.depth, "num_samples": current_node.num_samples}, + mark_exception=True + ) # # latent factors @@ -723,10 +727,24 @@ def _process_node(self, Ks, # leaf node based on depth limit or single cluster or all samples in same cluster if ((current_node.depth >= self.depth) and self.depth > 0) or current_node.k == 1 or n_clusters == 1: - current_node.leaf = True - pickle_path = os.path.join(f'{node_save_path}', f'node_{current_node.node_name}.p') - pickle.dump(current_node, open(pickle_path, "wb")) - return {"name":node_name, "target_jobs":[], "node_save_path":pickle_path} + stop_details = { + "depth": current_node.depth, + "depth_limit": self.depth, + "predicted_k": current_node.k, + "n_clusters_observed": n_clusters + } + # Pick a specific reason to make the text friendlier: + if (current_node.depth >= self.depth) and self.depth > 0: + reason = "depth limit reached" + elif current_node.k == 1: + reason = "predicted k == 1 (no further structure)" + elif n_clusters == 1: + reason = "all samples assigned to one cluster" + else: + reason = "stopping condition met" + + return self._finalize_leaf(current_node, node_save_path, reason=reason, details=stop_details) + # # go through each topic/cluster @@ -1110,4 +1128,56 @@ def _resolve_path(self, original_node_path: str): # 2. Combine it with the new experiment base new_node_path = new_experiment_base / relative_path - return new_node_path \ No newline at end of file + return new_node_path + + + """ + DEBUG FUNCTIONS + ---------------- + """ + def _write_stop_reason(self, node_save_path: str, reason: str, details: dict | None = None): + """ + Write a plain-text explanation for why decomposition did not proceed. + Creates /stop_reason.txt next to the node pickle. + """ + try: + reason_path = os.path.join(node_save_path, "stop_reason.txt") + with open(reason_path, "w") as f: + f.write(f"reason: {reason}\n") + if details: + for k, v in details.items(): + f.write(f"{k}: {v}\n") + except Exception as e: + # Don't break the run just because we couldn't write the note + warnings.warn(f"Could not write stop_reason.txt at {node_save_path}: {e}") + + def _finalize_leaf(self, current_node, node_save_path: str, reason: str, details: dict | None = None, *, + mark_exception: bool = False): + """ + Common exit path for any condition that stops further decomposition. + - marks leaf/exception on the node + - persists the node + - writes stop_reason.txt with a friendly explanation + - returns the scheduler payload + """ + current_node.leaf = True + current_node.exception = bool(mark_exception) + pickle_path = os.path.join(str(node_save_path), f'node_{current_node.node_name}.p') + + # persist the node + pickle.dump(current_node, open(pickle_path, "wb")) + + # write a human-readable reason file + # self._write_stop_reason(node_save_path, reason, details) + + + details = details or {} + details.update({ + "cluster_on": self.cluster_on, + "num_samples": current_node.num_samples, + "len_original_indices": len(current_node.original_indices), + }) + details.update(getattr(current_node, "user_node_data", {}) or {}) + self._write_stop_reason(node_save_path, reason, details) + + return {"name": current_node.node_name, "target_jobs": [], "node_save_path": pickle_path} diff --git a/TELF/pipeline/__init__.py b/TELF/pipeline/__init__.py index f0f242ba..94f7ca8a 100644 --- a/TELF/pipeline/__init__.py +++ b/TELF/pipeline/__init__.py @@ -65,3 +65,5 @@ from .blocks.ocelot_filter_block import OcelotFilterBlock from .blocks.auto_bunny_simple_block import AutoBunnySimpleBlock from .blocks.term_table_block import TermTableBlock +from .blocks.spacey_NER_block import SpacyNERBlock + diff --git a/TELF/pipeline/block_manager.py b/TELF/pipeline/block_manager.py index 032a304b..8eb86d25 100644 --- a/TELF/pipeline/block_manager.py +++ b/TELF/pipeline/block_manager.py @@ -359,7 +359,7 @@ def save_settings(self) -> None: for idx, blk in enumerate(self.blocks): fn = saved_dir / f"{idx}_{blk.__class__.__name__}.json" - fn.write_text(jsonpickle.encode(blk), encoding="utf-8") + fn.write_text(jsonpickle.encode(blk, keys=True), encoding="utf-8") def load_saved_settings(self) -> None: """ diff --git a/TELF/pipeline/blocks/__init__.py b/TELF/pipeline/blocks/__init__.py index ff6eb888..8a71656b 100644 --- a/TELF/pipeline/blocks/__init__.py +++ b/TELF/pipeline/blocks/__init__.py @@ -58,4 +58,5 @@ from .peacock_stats_block import PeacockStatsBlock from .ocelot_filter_block import OcelotFilterBlock from .auto_bunny_simple_block import AutoBunnySimpleBlock -from .term_table_block import TermTableBlock \ No newline at end of file +from .term_table_block import TermTableBlock +from .spacey_NER_block import SpacyNERBlock \ No newline at end of file diff --git a/TELF/pipeline/blocks/sbatch_block.py b/TELF/pipeline/blocks/sbatch_block.py index a349dcde..8a58e407 100644 --- a/TELF/pipeline/blocks/sbatch_block.py +++ b/TELF/pipeline/blocks/sbatch_block.py @@ -35,13 +35,13 @@ class SBatchBlock(AnimalBlock): Behaviour --------- - • **Skip submission** if every declared checkpoint already exists. + • **Skip submission** if every declared checkpoint already exists. • Otherwise: 1. create a staging dir `/` (spaces → `_`); 2. serialize the current bundle + wrapped block there; - 3. write a `run_block.py` runner (faulthandler enabled); - 4. write a hardened `run.sbatch` (threads pinned, Arrow off); - 5. `sbatch run.sbatch` and `sys.exit` so the pipeline resumes later. + 3. write a `run_block.py` runner (faulthandler + hardened jsonpickle); + 4. write a hardened `run.sbatch` (threads pinned, Arrow off, PYTHONPATH set); + 5. `sbatch run.sbatch` and exit so the pipeline resumes later. """ def __init__( self, @@ -58,7 +58,7 @@ def __init__( call_settings: Dict[str, Any] | None = None, **kw, ) -> None: - tag=wrapped_block.tag + tag = wrapped_block.tag super().__init__( needs=needs or wrapped_block.needs, provides=provides or wrapped_block.provides, @@ -101,7 +101,7 @@ def run(self, bundle: DataBundle) -> None: done = workdir / "_complete.json" if done.exists(): # runner wrote out a small dict: { provide_name: value, ... } - resumed: dict = jsonpickle.decode(done.read_text()) + resumed: dict = jsonpickle.decode(done.read_text(), keys=True) wrapped_tag = self.wrapped_block.tag for p in self.provides: ns = f"{wrapped_tag}.{p}" @@ -112,33 +112,113 @@ def run(self, bundle: DataBundle) -> None: # ensure SAVE_DIR is absolute bundle[SAVE_DIR_BUNDLE_KEY] = str(base_dir) - # 3) serialize bundle + block + # 3) serialize bundle + block (keys=True preserves non-string dict keys) (workdir / "input_bundle.json").write_text( - jsonpickle.encode(bundle), "utf-8" + jsonpickle.encode(bundle, keys=True), encoding="utf-8" ) (workdir / "block.json").write_text( - jsonpickle.encode(self.wrapped_block), "utf-8" + jsonpickle.encode(self.wrapped_block, keys=True), encoding="utf-8" ) + # sidecar with import path for robust re-instantiation + block_cls_path = f"{self.wrapped_block.__class__.__module__}:{self.wrapped_block.__class__.__qualname__}" + (workdir / "block_class.txt").write_text(block_cls_path, encoding="utf-8") + + # 4) runner script (Template placeholders; adds sys.path; spaCy Table handler; dict→class fallback) + from string import Template - # 4) runner script runner = workdir / "run_block.py" - runner.write_text(textwrap.dedent(f""" - import faulthandler, jsonpickle - faulthandler.enable() + project_root = Path.cwd() # adjust if your repo root differs at submit time + + tpl = Template(""" +import sys, importlib, types, faulthandler, jsonpickle +faulthandler.enable() + +# Make sure we can import your package/modules +sys.path.insert(0, $PROJECT_ROOT) + +# ---- jsonpickle handler: spaCy Table needs constructor run before inserts ---- +from jsonpickle.handlers import BaseHandler +try: + from spacy.lookups import Table as _SpaCyTable + _TABLES = [_SpaCyTable] +except Exception: + _TABLES = [] + +class _TableHandler(BaseHandler): + def flatten(self, obj, data): + data['py/object'] = obj.__class__.__module__ + '.' + obj.__class__.__name__ + data['items'] = list(obj.items()) + return data + + def restore(self, data): + cls = self.restore_class(data) + inst = cls() # ensure __init__ runs (creates .bloom, etc.) + if 'items' in data: + kv = data['items'] + else: + state = data.get('py/state', {}) + if isinstance(state, dict): + mapping = state.get('data', state) + else: + mapping = {} + kv = list(getattr(mapping, 'items', lambda: [])()) + if kv: + inst.update(dict(kv)) + return inst - bundle = jsonpickle.decode(open(r"{workdir/'input_bundle.json'}").read()) - block = jsonpickle.decode(open(r"{workdir/'block.json'}").read()) - out_bundle = block(bundle) +for _T in _TABLES: + jsonpickle.handlers.register(_T, _TableHandler) - # collect just the wrapped block's provides - result = {{}} - for p in {list(self.provides)!r}: - result[p] = out_bundle[f"{{block.tag}}.{{p}}"] +# ---- decode inputs (keys=True for non-string keys elsewhere) ---- +with open($INPUT_JSON, "r", encoding="utf-8") as _f: + bundle = jsonpickle.decode(_f.read(), keys=True) +with open($BLOCK_JSON, "r", encoding="utf-8") as _f: + block = jsonpickle.decode(_f.read(), keys=True) - open(r"{done}", "w").write(jsonpickle.encode(result)) - """).strip(), "utf-8") +# If import failed and we got a dict, rebuild the class instance using the sidecar +if isinstance(block, dict): + with open($BLOCK_CLASS_TXT, "r", encoding="utf-8") as _f: + cls_path = _f.read().strip() + mod_name, qualname = cls_path.split(":", 1) + mod = importlib.import_module(mod_name) + cls = mod + for part in qualname.split("."): + cls = getattr(cls, part) + # instantiate and copy state (skip framework-managed attrs) + NON_STATE = {"tag", "needs", "provides", "conditional_needs", "load_checkpoint"} + inst = cls() + for k, v in block.items(): + if k not in NON_STATE: + setattr(inst, k, v) + # keep tag/needs/provides if present in the dict + for k in ("tag", "needs", "provides", "conditional_needs"): + if k in block: + setattr(inst, k, block[k]) + block = inst + +# Run the block +out_bundle = block(bundle) + +# collect just the wrapped block's provides +result = {} +for p in $PROVIDES: + result[p] = out_bundle[block.tag + "." + p] + +with open($DONE_JSON, "w", encoding="utf-8") as _f: + _f.write(jsonpickle.encode(result, keys=True)) +""") + + script = tpl.substitute( + PROJECT_ROOT=repr(str(project_root)), + INPUT_JSON=repr(str(workdir / "input_bundle.json")), + BLOCK_JSON=repr(str(workdir / "block.json")), + BLOCK_CLASS_TXT=repr(str(workdir / "block_class.txt")), + DONE_JSON=repr(str(done)), + PROVIDES=repr(list(self.provides)), + ) + runner.write_text(textwrap.dedent(script), encoding="utf-8") - # 5) sbatch script + # 5) sbatch script (export PYTHONPATH to make imports resolvable in the job) sbatch = workdir / "run.sbatch" lines = [ "#!/bin/bash", @@ -155,11 +235,12 @@ def run(self, bundle: DataBundle) -> None: "export PANDAS_ARROW_DISABLED=1", "", f"cd {shquote(workdir)}", + f'export PYTHONPATH={shquote(project_root)}:"$PYTHONPATH"', ] match self.venv_type: case "venv": - lines.append(f"source {shquote(Path(self.venv_path).resolve()/ 'bin'/'activate')}") + lines.append(f"source {shquote(Path(self.venv_path).resolve() / 'bin' / 'activate')}") case "conda": lines += [ "source $(conda info --base)/etc/profile.d/conda.sh", @@ -174,7 +255,7 @@ def run(self, bundle: DataBundle) -> None: raise ValueError(f"Unknown venv_type {self.venv_type!r}") lines.append(f"python -Xfaulthandler {shquote(runner)}") - sbatch.write_text("\n".join(lines), "utf-8") + sbatch.write_text("\n".join(lines), encoding="utf-8") # 6) submit and exit print(f"🚀 {self.tag}: submitting via sbatch …") diff --git a/TELF/pipeline/blocks/semantic_hnmfk_block.py b/TELF/pipeline/blocks/semantic_hnmfk_block.py index eea6412f..d842f170 100644 --- a/TELF/pipeline/blocks/semantic_hnmfk_block.py +++ b/TELF/pipeline/blocks/semantic_hnmfk_block.py @@ -7,6 +7,8 @@ import numpy as np import pandas as pd import scipy.sparse as ss +import traceback +import warnings from ...factorization import HNMFk from ...pre_processing import Beaver @@ -55,12 +57,24 @@ def __call__(self, original_idx: np.ndarray): save_path=None, ) return X.T.tocsr(), {"vocab": vocab} - except Exception: - return ss.csr_matrix([[1]]), { - "stop_reason": "documents_words could not build matrix", + # except Exception: + # return ss.csr_matrix([[1]]), { + # "stop_reason": "documents_words could not build matrix", + # } + except Exception as e: + tb = traceback.format_exc() + warnings.warn(f"documents_words failed on {len(original_idx)} docs: {e}\n{tb}") + + # Preserve the docs dimension so you can see it's not 1 + n_docs = int(len(original_idx)) + placeholder = ss.csr_matrix((1, n_docs)) # 1 feature × N docs for H-mode + return placeholder, { + "stop_reason": "documents_words failed", + "exception": repr(e), + "traceback": tb, + "n_docs": n_docs, } - # ------------------------------------------------------------------ # # Main block # # ------------------------------------------------------------------ # diff --git a/TELF/pipeline/blocks/spacey_NER_block.py b/TELF/pipeline/blocks/spacey_NER_block.py new file mode 100644 index 00000000..862afcbf --- /dev/null +++ b/TELF/pipeline/blocks/spacey_NER_block.py @@ -0,0 +1,213 @@ +from pathlib import Path +from typing import Any, Dict, List, Optional +import json +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +class SpacyNERBlock(AnimalBlock): + """ + spaCy NER over one or more text columns (default: ['title', 'abstract']). + + For each specified text column `col`, adds: + - f"{col}_ents": JSON list of dicts {text, label, start, end} + - f"{col}_ents_by_label": JSON dict of {label: [unique entity strings]} + + Artifacts: + - /spaceyNER/spaceyNER.csv (enriched DataFrame) + - /spaceyNER/entities.csv (optional, exploded entity rows) + + Requirements: + - spaCy with a model installed (default: 'en_core_web_sm'). + + Parameters + ---------- + needs : tuple + Bundle keys to read. Default: ("df",). + provides : tuple + Bundle keys to write. Default: ("df", "ents_table"). + If you pass only ("df",), the exploded entities table is skipped. + text_columns : Optional[List[str]] + Which DF columns to run NER on. Default: ["title", "abstract"]. + id_field : str + Row identifier, used in the exploded entities table. Default: "eid". + spacy_model : str + spaCy model name to load. Default: "en_core_web_sm". + batch_size : int + spaCy pipe batch size. Default: 256. + n_process : int + Number of processes for spaCy pipe. Default: 1. + drop_existing : bool + If True, drop and rebuild any existing NER output columns. Default: True. + tag : str + Block tag and artifact folder name. Default: "spaceyNER". + init_settings : Optional[Dict[str, Any]] + Extra init settings; merged into defaults. + """ + + CANONICAL_NEEDS = ("df",) + + def __init__( + self, + *, + needs=CANONICAL_NEEDS, + provides=("df", "ents_table"), + text_columns: Optional[List[str]] = None, + id_field: str = "eid", + spacy_model: str = "en_core_web_lg", + batch_size: int = 256, + n_process: int = 1, + drop_existing: bool = True, + tag: str = "spaceyNER", + init_settings: Optional[Dict[str, Any]] = None, + **kw, + ): + self.id_field = id_field + + default_init = { + "text_columns": text_columns or ["title", "abstract"], + "spacy_model": spacy_model, + "batch_size": int(batch_size), + "n_process": int(n_process), + "drop_existing": bool(drop_existing), + "verbose": True, + } + + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings={}, # not used here + **kw, + ) + + # ------------------------------- helpers + + def _load_spacy(self): + try: + import spacy # local import so module is optional until needed + except Exception as e: + raise RuntimeError( + f"[{self.tag}] spaCy is not installed. Please install spaCy and a model." + ) from e + + model = self.init_settings["spacy_model"] + try: + # Disable components we don't need for speed. + nlp = spacy.load(model, disable=["tagger", "lemmatizer", "textcat"]) + except OSError as e: + raise RuntimeError( + f"[{self.tag}] spaCy model '{model}' is not installed.\n" + f"Install it with: python -m spacy download {model}" + ) from e + return nlp + + # ------------------------------- run + + def run(self, bundle: DataBundle) -> None: + # 1) Load input DF + df: pd.DataFrame = self.load_path(bundle[self.needs[0]]) + print(f"\n[{self.tag}] ====================================================") + print(f"[{self.tag}] df.shape = {df.shape}") + + text_cols: List[str] = list(self.init_settings.get("text_columns", ["title", "abstract"])) + present = [c for c in text_cols if c in df.columns] + missing = [c for c in text_cols if c not in df.columns] + if not present: + raise ValueError(f"[{self.tag}] None of the requested text columns are present: {text_cols}") + if missing: + print(f"[{self.tag}] WARNING: missing text columns {missing}; will skip them") + + # 2) Output dir + root = Path(bundle.get(SAVE_DIR_BUNDLE_KEY, ".")) + out = root / self.tag + out.mkdir(parents=True, exist_ok=True) + print(f"[{self.tag}] output dir = {out}") + + # 3) Load spaCy + nlp = self._load_spacy() + batch_size = int(self.init_settings.get("batch_size", 256)) + n_process = int(self.init_settings.get("n_process", 1)) + drop_existing = bool(self.init_settings.get("drop_existing", True)) + + # 4) Prepare destination columns; optionally drop existing + dest_cols = [] + for col in present: + dest_cols += [f"{col}_ents", f"{col}_ents_by_label"] + if drop_existing: + to_drop = [c for c in dest_cols if c in df.columns] + if to_drop: + df = df.drop(columns=to_drop) + print(f"[{self.tag}] dropped existing NER columns: {to_drop}") + + df_proc = df.copy() + exploded_rows: List[Dict[str, Any]] = [] + + # 5) Run NER column-by-column + for col in present: + print(f"[{self.tag}] NER on column = {col}") + texts = df_proc[col].fillna("").astype(str).tolist() + ents_json_col: List[str] = [] + bylabel_json_col: List[str] = [] + + i_to_row_id = ( + df_proc[self.id_field].tolist() if self.id_field in df_proc.columns else list(range(len(df_proc))) + ) + + for i, doc in enumerate(nlp.pipe(texts, batch_size=batch_size, n_process=n_process)): + ents = [ + { + "text": ent.text, + "label": ent.label_, + "start": int(ent.start_char), + "end": int(ent.end_char), + } + for ent in doc.ents + ] + + by_label: Dict[str, List[str]] = {} + for ent in doc.ents: + lst = by_label.setdefault(ent.label_, []) + if ent.text not in lst: + lst.append(ent.text) + + ents_json_col.append(json.dumps(ents, ensure_ascii=False)) + bylabel_json_col.append(json.dumps(by_label, ensure_ascii=False)) + + # Collect exploded rows + rid = i_to_row_id[i] + for ent in doc.ents: + exploded_rows.append( + { + self.id_field: rid, + "source_column": col, + "text": ent.text, + "label": ent.label_, + "start": int(ent.start_char), + "end": int(ent.end_char), + } + ) + + df_proc[f"{col}_ents"] = ents_json_col + df_proc[f"{col}_ents_by_label"] = bylabel_json_col + + # 6) Save artifacts + register + df_path = out / f"{self.tag}.csv" + df_proc.to_csv(df_path, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[0], df_path) + bundle[f"{self.tag}.{self.provides[0]}"] = df_proc + print(f"[{self.tag}] saved df → {df_path}") + + # optional exploded entities table + if len(self.provides) > 1: + ents_df = pd.DataFrame( + exploded_rows, + columns=[self.id_field, "source_column", "text", "label", "start", "end"], + ) + table_path = out / "entities.csv" + ents_df.to_csv(table_path, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[1], table_path) + bundle[f"{self.tag}.{self.provides[1]}"] = ents_df + print(f"[{self.tag}] saved entities → {table_path}") diff --git a/examples/Full TELF Pipeline/single_block_examples/spacey_ner_block_example.ipynb b/examples/Full TELF Pipeline/single_block_examples/spacey_ner_block_example.ipynb new file mode 100644 index 00000000..58e8651c --- /dev/null +++ b/examples/Full TELF Pipeline/single_block_examples/spacey_ner_block_example.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "29d4ec69", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/barron/anaconda3/envs/TELF/lib/python3.11/site-packages/pymilvus/client/__init__.py:6: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", + " from pkg_resources import DistributionNotFound, get_distribution\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "from pathlib import Path\n", + "\n", + "from TELF.pipeline import BlockManager \n", + "from TELF.pipeline.blocks import (\n", + " DataBundle,\n", + " LoadTermsBlock,\n", + " TermAttributionBlock,\n", + " VultureCleanBlock,\n", + " SAVE_DIR_BUNDLE_KEY,\n", + " DIR_LIST_BUNDLE_KEY,\n", + " RESULTS_DEFAULT,\n", + " SOURCE_DIR_BUNDLE_KEY,\n", + " SpacyNERBlock\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "06581eaf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[spaceyNER] needs → (df) provides → (df, ents_table)\n" + ] + } + ], + "source": [ + "ner_block = SpacyNERBlock(model_name=\"en_core_web_lg\", text_column=\"abstract\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a2a778d", + "metadata": {}, + "outputs": [], + "source": [ + "pipeline_blocks = [ner_block]\n", + "\n", + "bundle = DataBundle({\n", + " SOURCE_DIR_BUNDLE_KEY: Path(\"..\") / \"..\" / \"..\" / \"data\" ,\n", + " SAVE_DIR_BUNDLE_KEY: Path(\"example_results\") / \"ner_block_example\" ,\n", + " 'df': pd.read_csv(Path(\"..\") / \"..\" / \"..\" / \"data\" / \"sample2.csv\").head(50)\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e340948b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Terms – all needs met
VultureClean – all needs met
Attribution – all needs met" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Block (tag) │ Needs (✓/✗) │ Provides\n", + "───────────────────────────────────────────────────────────\n", + "LoadTermsBlock (Terms) │ dir │ ['terms', 'substitutions', 'substitutions_reverse', 'query']\n", + "VultureCleanBlock (VultureClean) │ df │ ['df', 'vulture_steps']\n", + "TermAttributionBlock (Attribution) │ df, terms │ ['df', 'term_representation_df']\n", + "\n", + "▶ [1/3] Terms …\n", + "✓ [1/3] Terms finished in 1.89s\n", + "▶ [2/3] VultureClean …\n", + "✓ [2/3] VultureClean finished in 71.72s\n", + "▶ [3/3] Attribution …\n", + "✓ [3/3] Attribution finished in 0.11s\n" + ] + } + ], + "source": [ + "manager = BlockManager(pipeline_blocks, databundle=bundle)\n", + "bundle = manager()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "TELF", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/Lynx/README.md b/examples/Lynx/README.md index 0f8d5d3f..3bca3c4a 100644 --- a/examples/Lynx/README.md +++ b/examples/Lynx/README.md @@ -1,6 +1,18 @@ ## Usage: -1. ```cd``` into the root directory of this repository (```cd TELF```). +1. ```cd``` into the root directory of this repository + + ```cd TELF``` 2. Create a ```projects``` directory. + + ```mkdir projects``` 3. Put your post-processing folder for each project under this directory. -4. On terminal run ```streamlit run TELF/applications/Lynx/frontend/main.py``` -5. Have fun! \ No newline at end of file + + ```cp -r /path/to/project1 TELF/projects``` +4. On terminal start the server + + ```streamlit run TELF/applications/Lynx/frontend/main.py``` +5. **Optional** if running on a remote server, forward the ports by running the following on the local terminal + + ```ssh USER@HOST -L 8501:localhost:8501``` +6. Have fun! + diff --git a/examples/Termite/01-Termite.ipynb b/examples/Termite/01-Termite.ipynb new file mode 100755 index 00000000..f7e6e965 --- /dev/null +++ b/examples/Termite/01-Termite.ipynb @@ -0,0 +1,597 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Imports and Paths" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from TELF.applications import Termite\n", + "from TELF.applications.Termite.neo4j_termite import ENTITY, RETURN_TYPE, ATTRIBUTES, ET,\\\n", + " YEAR_TYPE, FROM_COL, DOCUMENT_TYPE,\\\n", + " ROW_INDEX, ATTR_COL, ATTR_NAME, TT,\\\n", + " R, DOCUMENT_YEAR_RELATION, HT,\\\n", + " AUTHOR_DOCUMENT_RELATION, AUTHOR_ID_TYPE,\\\n", + " EXTRACT_H, DOCUMENT_CITES_RELATION,\\\n", + " DOCUMENT_CITED_RELATION, DOCUMENT_TYPE_SCOPUS,\\\n", + " EXTRACT_T, PAIRING, HEAD_TO_MANY\n", + "from copy import deepcopy\n", + "import pandas as pd\n", + "\n", + "username, password = \"neo4j\", \"local_password\"\n", + "token = None\n", + "URI = \"neo4j://localhost:7666\"\n", + "credentials = (URI, (username, password))\n", + "\n", + "\n", + "termite = Termite(kg_credentials=credentials, \n", + " vector_uri=\"http://localhost:19530\",\n", + " db_nme=\"default\",\n", + " token=token,\n", + " verbose=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Paths for source and destination data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "raw_csv_path = \"../../data/sample2.csv\"\n", + "triplets_path = \"./01_termite_output/sample_triplets_data.csv\"\n", + "import os\n", + "os.makedirs('01_termite_output', exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NEO4J INJECTION" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Extraction Functions for each of the relations with complex data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def list_split_no_attrs(data, split_with=';'):\n", + " returnable_entities = []\n", + " if type(data) == str:\n", + " split_values = data.split(split_with)\n", + " for entity_value in split_values:\n", + " entity_returnable = deepcopy(RETURN_TYPE)\n", + " entity_returnable[ENTITY] = entity_value\n", + " returnable_entities.append(entity_returnable)\n", + " return returnable_entities\n", + " else:\n", + " return [deepcopy(RETURN_TYPE)]\n", + "\n", + "def get_cites(args):\n", + " data_string = args['data']\n", + " return list_split_no_attrs(data_string.citations)\n", + " \n", + "def get_cited(args):\n", + " data_string = args['data']\n", + " return list_split_no_attrs(data_string.references)\n", + " \n", + "def get_authors_ID(args):\n", + " data_string = args['data']\n", + "\n", + " data = data_string.s2_author_ids\n", + " if type(data) == str:\n", + " split_author_ids= data.split(';')\n", + " authors = data_string.s2_authors\n", + " if type(authors) == str:\n", + " authors_split_values = authors.split(';')\n", + " returnable_entities = []\n", + " \n", + " for entity_value, attribute in zip(split_author_ids, authors_split_values ):\n", + " entity_returnable = deepcopy(RETURN_TYPE)\n", + " entity_returnable[ENTITY] = entity_value\n", + " entity_returnable[ATTRIBUTES] = [('name', attribute)]\n", + " returnable_entities.append(entity_returnable)\n", + " return returnable_entities\n", + " else:\n", + " return [deepcopy(RETURN_TYPE)]\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define the mapping from the columns to the desired structure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"\n", + "Entities need to tell the type, the column it comes from, any associates attributes and how to get those attribuetes if embedded in a data structure inline.\n", + "Relations need to say the head type, relation type, tail type, and how to get the tail if the tail is compossed of multiple entities embedded inline. \n", + "The fourth passed item is the tail entity retrieval, and should return a list of dictionaries, where each dictionary contains: 'entity','weight','attributes'\n", + "\"\"\"\n", + "column_triplet_map = {\n", + " 'ENTITIES':\n", + " [\n", + " {ET:YEAR_TYPE, FROM_COL: 'year'},\n", + " {ET:DOCUMENT_TYPE, FROM_COL: ROW_INDEX, ATTR_COL:[{FROM_COL: 'title', ATTR_NAME:'Title', },\n", + " {FROM_COL: 's2id', ATTR_NAME:'S2ID', },\n", + " {FROM_COL: 'doi', ATTR_NAME:'DOI', },\n", + " ]},\n", + " ],\n", + " 'RELATIONS':\n", + " [ \n", + " {HT:DOCUMENT_TYPE, R:DOCUMENT_YEAR_RELATION, TT:YEAR_TYPE},\n", + " {HT:AUTHOR_ID_TYPE, R:AUTHOR_DOCUMENT_RELATION, TT:DOCUMENT_TYPE, EXTRACT_H: get_authors_ID},\n", + " {HT:DOCUMENT_TYPE, R:DOCUMENT_CITES_RELATION, TT:DOCUMENT_TYPE_SCOPUS, EXTRACT_T: get_cites, PAIRING: HEAD_TO_MANY},\n", + " {HT:DOCUMENT_TYPE, R:DOCUMENT_CITED_RELATION, TT:DOCUMENT_TYPE_SCOPUS, EXTRACT_T: get_cited, PAIRING: HEAD_TO_MANY},\n", + " ]\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Construct Triplets from Raw Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "termite.from_csv_to_triplets(raw_csv_path, triplets_path, column_triplet_map)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inject Triplets into graph" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "termite.update_database_multithreaded(triplets_path)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MILVUS INJECTION\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/barron/anaconda3/envs/TELF/lib/python3.11/site-packages/pymilvus/client/__init__.py:6: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", + " from pkg_resources import DistributionNotFound, get_distribution\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Top-3 hits:\n", + " score=1.0000 id=0 text=None\n", + " score=0.9441 id=213 text=None\n", + " score=0.9423 id=58 text=None\n" + ] + } + ], + "source": [ + "# test_opensearch_termite.py\n", + "import os\n", + "import pandas as pd\n", + "\n", + "from TELF.applications import Termite\n", + "\n", + "# --- ensure backend is OpenSearch (this is the default anyway) ---\n", + "os.environ[\"EMBEDDING_STORE\"] = \"opensearch\"\n", + "os.environ[\"OS_HOST\"] = \"localhost\"\n", + "os.environ[\"OS_PORT\"] = \"9200\"\n", + "os.environ[\"OS_USE_SSL\"] = \"false\"\n", + "\n", + "# --- init Termite (Neo4j creds not required for this test) ---\n", + "t = Termite(\n", + " kg_credentials=None,\n", + " verbose=True,\n", + " # you can swap the model if you want; default is \"malteos/scincl\"\n", + " model_name=\"malteos/scincl\",\n", + ")\n", + "\n", + "# --- load a small dataframe ---\n", + "df = pd.read_csv(raw_csv_path)\n", + "\n", + "# --- compute embeddings (returns dict: index -> vector) ---\n", + "emb_map = t.compute_embeddings(df, model_name=\"malteos/scincl\")\n", + "# convert to row-ordered list to align with df rows\n", + "embeddings = [emb_map[i] for i in df.index.tolist()]\n", + "\n", + "# infer embedding dimension from the first row\n", + "dim = len(embeddings[0])\n", + "\n", + "index_name = \"termite_vectors_test\"\n", + "\n", + "# --- create the OpenSearch k-NN index (HNSW, cosine by default) ---\n", + "t.make_vector_schema(collection_name=index_name, dim=dim, metric=\"cosine\")\n", + "\n", + "# --- shape data & upsert ---\n", + "data = t.df_to_data(\n", + " embeddings=embeddings,\n", + " df_path=raw_csv_path,\n", + " columns_collection_map={\n", + " # map CSV columns into the payload keys expected by the store\n", + " \"id\": \"eid\",\n", + " \"text\": \"abstract\",\n", + " },\n", + ")\n", + "t.inject_vectors(index_name, data)\n", + "\n", + "# --- quick search: use the first vector as the query ---\n", + "hits = t.search_vectors(index_name, embeddings[0], k=3)\n", + "print(\"\\nTop-3 hits:\")\n", + "for _id, score, payload in hits:\n", + " print(f\" score={score:.4f} id={_id} text={payload.get('text')}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Top-3 hits:\n", + " score=1.0000 id=7fac733d-83ec-4b15-a48d-16a893a2373a text=Zero-day vulnerabilities pose a significant threat to cybersecurity systems. The kernel trick in SVMs enables efficient \n", + " score=0.9441 id=5bf26de5-b5da-499c-bcb2-d70ae542792f text=Zero-day vulnerabilities pose a significant threat to cybersecurity systems. Graph neural networks excel at processing s\n", + " score=0.9423 id=83913121-bc9c-4b12-ac93-5563ebd881a4 text=Reinforcement learning enables agents to learn optimal policies through trial and error. Highly specific datasets of sci\n", + "\n", + "Stored _source keys on a sample doc: ['id', 'embedding', 'text']\n" + ] + } + ], + "source": [ + "# test_opensearch_termite_complete.py\n", + "\"\"\"\n", + "End-to-end OpenSearch + TELF Termite example (OpenSearch 2.13)\n", + "\n", + "- Uses Termite to compute embeddings (malteos/scincl, 768-D).\n", + "- Creates a knn_vector index (HNSW, cosine).\n", + "- Upserts docs with id + embedding + text (top-level \"_source.text\").\n", + "- Queries with the OpenSearch-native k-NN body and prints text.\n", + "\n", + "Requirements:\n", + " pip install pandas opensearch-py TELF\n", + "\n", + "Notes:\n", + " - Set raw_csv_path to your CSV.\n", + " - CSV should have an 'eid' column (stringable doc id) and an 'abstract' column (text).\n", + " - If your column names differ, change ID_COL/TEXT_COL below.\n", + "\"\"\"\n", + "\n", + "import os\n", + "import warnings\n", + "import pandas as pd\n", + "from typing import List, Dict\n", + "\n", + "# Suppress the pymilvus pkg_resources deprecation warning that TELF pulls in\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning, message=\"pkg_resources is deprecated\")\n", + "\n", + "from TELF.applications import Termite\n", + "from opensearchpy import OpenSearch, helpers\n", + "\n", + "# ---------- USER CONFIG ----------\n", + "INDEX_NAME = \"termite_vectors_test\"\n", + "ID_COL = \"eid\" # <-- change if your id column has a different name\n", + "TEXT_COL = \"abstract\" # <-- change if your text column has a different name\n", + "MODEL_NAME = \"malteos/scincl\" # 768-D sentence embeddings\n", + "RECREATE_INDEX = True # set False to append\n", + "TOP_K = 3\n", + "# ---------------------------------\n", + "\n", + "# OpenSearch connection (dev defaults)\n", + "OS_HOST = os.environ.get(\"OS_HOST\", \"localhost\")\n", + "OS_PORT = int(os.environ.get(\"OS_PORT\", \"9200\"))\n", + "OS_USE_SSL = os.environ.get(\"OS_USE_SSL\", \"false\").lower() == \"true\"\n", + "\n", + "def get_client() -> OpenSearch:\n", + " return OpenSearch(\n", + " hosts=[{\"host\": OS_HOST, \"port\": OS_PORT}],\n", + " use_ssl=OS_USE_SSL,\n", + " verify_certs=False,\n", + " http_compress=True,\n", + " timeout=60,\n", + " )\n", + "\n", + "def ensure_index(client: OpenSearch, index: str, dim: int, recreate: bool = False) -> None:\n", + " if recreate and client.indices.exists(index=index):\n", + " client.indices.delete(index=index, ignore=[404])\n", + "\n", + " if client.indices.exists(index=index):\n", + " return\n", + "\n", + " body = {\n", + " \"settings\": {\n", + " \"index\": {\n", + " \"knn\": True,\n", + " \"number_of_shards\": 1,\n", + " \"number_of_replicas\": 0,\n", + " # Optional: slightly higher ef_search for better recall (set at index level on OS 2.13)\n", + " \"knn.algo_param.ef_search\": 128,\n", + " }\n", + " },\n", + " \"mappings\": {\n", + " \"properties\": {\n", + " \"id\": {\"type\": \"keyword\"},\n", + " \"text\": {\"type\": \"text\"},\n", + " \"metadata\": {\"type\": \"object\"},\n", + " \"embedding\": {\n", + " \"type\": \"knn_vector\",\n", + " \"dimension\": int(dim),\n", + " \"method\": {\n", + " \"name\": \"hnsw\",\n", + " \"engine\": \"nmslib\",\n", + " \"space_type\": \"cosinesimil\"\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " client.indices.create(index=index, body=body)\n", + "\n", + "def to_float_list(vec) -> List[float]:\n", + " # Normalize numpy arrays / tensors, ensure native floats\n", + " try:\n", + " import numpy as np\n", + " if isinstance(vec, np.ndarray):\n", + " vec = vec.tolist()\n", + " except Exception:\n", + " pass\n", + " return [float(x) for x in vec]\n", + "\n", + "def bulk_upsert(client: OpenSearch, index: str, ids: List[str], vectors: List[List[float]], texts: List[str]):\n", + " actions = []\n", + " for i, vid in enumerate(ids):\n", + " actions.append({\n", + " \"_op_type\": \"index\",\n", + " \"_index\": index,\n", + " \"_id\": str(vid),\n", + " \"_source\": {\n", + " \"id\": str(vid),\n", + " \"embedding\": to_float_list(vectors[i]),\n", + " \"text\": str(texts[i]),\n", + " }\n", + " })\n", + " helpers.bulk(client, actions)\n", + " client.indices.refresh(index=index)\n", + "\n", + "def knn_search(client: OpenSearch, index: str, query_vec: List[float], k: int = 3, source_fields=(\"id\",\"text\")) -> List[Dict]:\n", + " body = {\n", + " \"size\": int(k),\n", + " \"query\": {\n", + " \"knn\": {\n", + " # OpenSearch-native syntax: field name is the key; object has \"vector\" and \"k\"\n", + " \"embedding\": {\"vector\": to_float_list(query_vec), \"k\": int(k)}\n", + " }\n", + " },\n", + " \"_source\": list(source_fields)\n", + " }\n", + " resp = client.search(index=index, body=body)\n", + " return resp.get(\"hits\", {}).get(\"hits\", [])\n", + "\n", + "def main():\n", + " # 1) Load data\n", + " df = pd.read_csv(raw_csv_path)\n", + " if ID_COL not in df.columns or TEXT_COL not in df.columns:\n", + " raise ValueError(f\"CSV must have columns '{ID_COL}' and '{TEXT_COL}'. Found: {list(df.columns)}\")\n", + "\n", + " # 2) Compute embeddings with Termite (SciNCL by default)\n", + " t = Termite(kg_credentials=None, verbose=True, model_name=MODEL_NAME)\n", + "\n", + " # emb_map: row_index -> vector; make row-ordered list aligned with df\n", + " emb_map = t.compute_embeddings(df, model_name=MODEL_NAME)\n", + " embeddings = [emb_map[i] for i in df.index.tolist()]\n", + " dim = len(embeddings[0])\n", + "\n", + " # 3) Connect to OpenSearch and prepare index\n", + " client = get_client()\n", + " ensure_index(client, INDEX_NAME, dim, recreate=RECREATE_INDEX)\n", + "\n", + " # 4) Prepare ids/texts and upsert with payloads\n", + " ids = df[ID_COL].astype(str).tolist()\n", + " texts = df[TEXT_COL].fillna(\"\").astype(str).tolist()\n", + " bulk_upsert(client, INDEX_NAME, ids, embeddings, texts)\n", + "\n", + " # 5) Query with the first vector\n", + " hits = knn_search(client, INDEX_NAME, embeddings[0], k=TOP_K, source_fields=(\"id\",\"text\"))\n", + "\n", + " print(\"\\nTop-{} hits:\".format(TOP_K))\n", + " for h in hits:\n", + " _id = h.get(\"_id\")\n", + " score = float(h.get(\"_score\", 0.0))\n", + " src = h.get(\"_source\", {})\n", + " text = src.get(\"text\")\n", + " print(f\" score={score:.4f} id={_id} text={text[:120]}\")\n", + "\n", + " # 6) (Optional) Inspect stored fields once\n", + " sample = client.search(index=INDEX_NAME, body={\"size\":1,\"query\":{\"match_all\":{}}})\n", + " keys = list(sample[\"hits\"][\"hits\"][0][\"_source\"].keys())\n", + " print(\"\\nStored _source keys on a sample doc:\", keys)\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Query: What problem in real-world malware labeling does the HNMFk Classifier aim to solve?\n", + "Top-5 hits:\n", + " score=0.8113 id=fafed8b2-197d-45f6-ad53-881821264fa4 text=Recurrent neural networks are widely used for sequential data such as speech and text. Adversarial machine learning explores methods to enhance model robustness against attacks. Training deep learning models requires high computational power and GPUs. We propose an efficient distributed out-of-memory implementation of the non-negative matrix factorization (NMF) algorithm for heterogeneous high-performance-computing systems. The proposed implementation is based on prior work on NMFk, which can perform automatic model selection and extract latent variables and patterns from data. In this work, we extend NMFk by adding support for dense and sparse matrix operation on multi-node, multi-GPU systems. The resulting algorithm is optimized for out-of-memory problems where the memory required to factorize a given matrix is greater than the available GPU memory. Memory complexity is reduced by batching/tiling strategies, and sparse and dense matrix operations are significantly accelerated with GPU cores (or tensor cores when available). Input/output latency associated with batch copies between host and device is hidden using CUDA streams to overlap data transfers and compute asynchronously, and latency associated with collective communications (both intra-node and inter-node) is reduced using optimized NVIDIA Collective Communication Library (NCCL) based communicators. Benchmark results show significant improvement, from 32X to 76x speedup, with the new implementation using GPUs over the CPU-based NMFk. Good weak scaling was demonstrated on up to 4096 multi-GPU cluster nodes with approximately 25,000 GPUs when decomposing a dense 340 Terabyte-size matrix and an 11 Exabyte-size sparse matrix of density 10−6. Training deep learning models requires high computational power and GPUs. Clustering algorithms like K-Means help in customer segmentation and anomaly detection. Reinforcement learning enables AI agents to optimize their decision-making through rewards and penalties. Feature selection plays a crucial role in improving model efficiency and reducing overfitting. Topic modeling is one of the key analytic techniques for organizing and analysis of large text corpora. One approach to topic modeling is the recently introduced SeNMFk, a method based on semantic non-negative matrix factorization (NMF) with automatic model determination (NMFk), where the text-document matrix and word-context (co-occurrence) matrix are jointly factorized. The text-document matrix is the term frequency-inverse document frequency (TF-IDF) matrix, and the word-context matrix represents the number of times two words co-occur in a pre-determined window of text. Incorporating the semantic structure of the text with the ability to estimate the number of topics enables a coherent separation of the latent topics and accurate document clustering. This approach, however, only identifies the highest level of topics or the main topics/themes. Many text corpora often include a very complex structure of sub-topics beyond the main themes. For example, a set of documents can be separated into main topics, such as, sports, politics, science, etc. Each of these main topics can be further separated into sub-topics. For example, sport theme can be separated to the subtopics tennis, soccer, football, etc. This process can be repeated by expanding the separation until finding all the sub-topics in the corpus. Here, we introduce a hierarchical SeNMFk approach, that can extract fine-grained sub-topics and their semantic sub-structures. By hierarchically applying SeNMFk, we break down the main topics and extract previously unknown sub-topics as well as the corresponding sub-semantic structures that can serve as narrow vocabularies – scientific-jargon seeds for local Name Entities Recognition (NER). We demonstrate our hierarchical SeNMFk method by performing topic modeling on all papers posted in arXiv, which is ~2 million+ papers. To enhance the semantic clustering in each topic, we also jointly factorize the category-text matrix, values of which represents the TF-IDF of tokens per document category. Here the categories are pre-determined/reported by the authors of the document based on its field of research in arXiv. Our results show the ability and practicality of our hierarchical SeNMFk to extract meaningful topics and find their semantic sub-structures from large datasets.\n", + " score=0.8035 id=6258761f-90e3-4f25-9481-ed66539f1401 text=Identification of the family to which a malware specimen belongs is essential in understanding the behavior of the malware and developing mitigation strategies. Solutions proposed by prior work, however, are often not practicable due to the lack of realistic evaluation factors. These factors include learning under class imbalance, the ability to identify new malware, and the cost of production-quality labeled data. In practice, deployed models face prominent, rare, and new malware families. At the same time, obtaining a large quantity of up-to-date labeled malware for training a model can be expensive. In this paper, we address these problems and propose a novel hierarchical semi-supervised algorithm, which we call the HNMFk Classifier, that can be used in the early stages of the malware family labeling process. Our method is based on non-negative matrix factorization with automatic model selection, that is, with an estimation of the number of clusters. With HNMFk Classifier, we exploit the hierarchical structure of the malware data together with a semi-supervised setup, which enables us to classify malware families under conditions of extreme class imbalance. Our solution can perform abstaining predictions, or rejection option, which yields promising results in the identification of novel malware families and helps with maintaining the performance of the model when a low quantity of labeled data is used. We perform bulk classification of nearly 2,900 both rare and prominent malware families, through static analysis, using nearly 388,000 samples from the EMBER-2018 corpus. In our experiments, we surpass both supervised and semi-supervised baseline models with an F1 score of 0.80. Anomaly detection techniques are widely used in fraud detection and cybersecurity. Explainable AI (XAI) enhances trust in AI models by making decisions interpretable. Cybersecurity frameworks like NIST provide guidelines for risk assessment and mitigation. Autoencoders are useful for dimensionality reduction and anomaly detection. Anomaly detection techniques are widely used in fraud detection and cybersecurity. Differential privacy ensures that machine learning models do not leak sensitive user data.\n", + " score=0.8034 id=885f68ca-ad59-4270-a5ff-c127c2895586 text=Autoencoders can effectively perform unsupervised feature learning and anomaly detection.Machine learning models require a substantial amount of data for training. Adversarial attacks can manipulate machine learning models by introducing subtle perturbations. Autoencoders can effectively perform unsupervised feature learning and anomaly detection.Machine learning models require a substantial amount of data for training. The kernel trick in SVMs enables efficient classification in non-linearly separable data.\n", + " score=0.8002 id=ed87567b-6463-4671-8cdd-52951831a0d0 text=The kernel trick in SVMs enables efficient classification in non-linearly separable data. Non-negative matrix factorization (NMF) with missing-value completion is a well-known effective Collaborative Filtering (CF) method used to provide personalized user recommendations. However, traditional CF relies on a privacy-invasive collection of user data to build a central recommender model. One-shot federated learning has recently emerged as a method to mitigate the privacy problem while addressing the traditional communication bottleneck of federated learning. In this paper, we present the first one-shot federated CF implementation, named One-FedCF, for groups of users or collaborating organizations. In our solution, the clients first apply local CF in-parallel to build distinct, client-specific recommenders. Then, the privacy-preserving local item patterns and biases from each client are shared with the processor to perform joint factorization in order to extract the global item patterns. Extracted patterns are then aggregated to each client to build the local models via information retrieval transfer. In our experiments, we demonstrate our approach with two MovieLens datasets and show results competitive with the state-of-the-art federated recommender systems at a substantial decrease in the number of communications. Multi-factor authentication enhances security by requiring multiple verification steps. Reinforcement learning enables agents to learn optimal policies through trial and error. Ensemble methods such as bagging and boosting combine multiple models to improve predictions. Blockchain technology enhances security and transparency in AI-driven systems.\n", + " score=0.7993 id=ce9fbed7-c072-4595-b43f-73d7889ca034 text=Recurrent neural networks are widely used for sequential data such as speech and text. Reinforcement learning enables agents to learn optimal policies through trial and error. Identification of the family to which a malware specimen belongs is essential in understanding the behavior of the malware and developing mitigation strategies. Solutions proposed by prior work, however, are often not practicable due to the lack of realistic evaluation factors. These factors include learning under class imbalance, the ability to identify new malware, and the cost of production-quality labeled data. In practice, deployed models face prominent, rare, and new malware families. At the same time, obtaining a large quantity of up-to-date labeled malware for training a model can be expensive. In this paper, we address these problems and propose a novel hierarchical semi-supervised algorithm, which we call the HNMFk Classifier, that can be used in the early stages of the malware family labeling process. Our method is based on non-negative matrix factorization with automatic model selection, that is, with an estimation of the number of clusters. With HNMFk Classifier, we exploit the hierarchical structure of the malware data together with a semi-supervised setup, which enables us to classify malware families under conditions of extreme class imbalance. Our solution can perform abstaining predictions, or rejection option, which yields promising results in the identification of novel malware families and helps with maintaining the performance of the model when a low quantity of labeled data is used. We perform bulk classification of nearly 2,900 both rare and prominent malware families, through static analysis, using nearly 388,000 samples from the EMBER-2018 corpus. In our experiments, we surpass both supervised and semi-supervised baseline models with an F1 score of 0.80. A confusion matrix provides a detailed breakdown of a classification model’s performance. Hyperparameter tuning is necessary to achieve optimal performance in machine learning models. Autoencoders are useful for dimensionality reduction and anomaly detection.\n" + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Top-3 hits:\n", + " score=1.0000 id=7fac733d-83ec-4b15-a48d-16a893a2373a text=Zero-day vulnerabilities pose a significant threat to cybersecurity systems. The kernel trick in SVMs enables efficient classification in non-linearly separable data. Graph neural networks excel at processing structured graph data for various applications. Support vector machines are effective in high-dimensional spaces for classification problems. Supervisory Control and Data Acquisition (SCADA) systems often serve as the nervous system for substations within power grids. These systems facilitate real-time monitoring, data acquisition, control of equipment, and ensure smooth and efficient operation of the substation and its connected devices. As the dependence on these SCADA systems grows, so does the risk of potential malicious intrusions that could lead to significant outages or even permanent damage to the grid. Previous work has shown that dimensionality reduction-based approaches, such as Principal Component Analysis (PCA), can be used for accurate identification of anomalies in SCADA systems. While not specifically applied to SCADA, non-negative matrix factorization (NMF) has shown strong results at detecting anomalies in wireless sensor networks. These unsupervised approaches model the normal or expected behavior and detect the unseen types of attacks or anomalies by identifying the events that deviate from the expected behavior. These approaches; however, do not model the complex and multi-dimensional interactions that are naturally present in SCADA systems. Differently, non-negative tensor decomposition is a powerful unsupervised machine learning (ML) method that can model the complex and multi-faceted activity details of SCADA events. In this work, we novelly apply the tensor decomposition method Canonical Polyadic Alternating Poisson Regression (CP-APR) with a probabilistic framework, which has previously shown state-of-the-art anomaly detection results on cyber network data, to identify anomalies in SCADA systems. We showcase that the use of statistical behavior analysis of SCADA communication with tensor decomposition improves the specificity and accuracy of identifying anomalies in electrical grid systems. In our experiments, we model real-world SCADA system data collected from the electrical grid operated by Los Alamos National Laboratory (LANL) which provides transmission and distribution service through a partnership with Los Alamos County, and detect synthetically generated anomalies. Ensemble methods such as bagging and boosting combine multiple models to improve predictions. Explainable AI (XAI) enhances trust in AI models by making decisions interpretable. Transfer learning allows pre-trained models to be fine-tuned for new tasks with limited data. The bias-variance tradeoff determines the model’s ability to generalize to new data. Ensemble methods such as bagging and boosting combine multiple models to improve predictions.\n", + " score=0.9441 id=5bf26de5-b5da-499c-bcb2-d70ae542792f text=Zero-day vulnerabilities pose a significant threat to cybersecurity systems. Graph neural networks excel at processing structured graph data for various applications. Highly specific datasets of scientific literature are important for both research and education. However, it is difficult to build such datasets at scale. A common approach is to build these datasets reductively by applying topic modeling on an established corpus and selecting specific topics. A more robust but time-consuming approach is to build the dataset constructively in which a subject matter expert (SME) handpicks documents. This method does not scale and is prone to error as the dataset grows. Here we showcase a new tool, based on machine learning, for constructively generating targeted datasets of scientific literature. Given a small initial “core” corpus of papers, we build a citation network of documents. At each step of the citation network, we generate text embeddings and visualize the embeddings through dimensionality reduction. Papers are kept in the dataset if they are “similar” to the core or are otherwise pruned through human-in-the-loop selection. Additional insight into the papers is gained through sub-topic modeling using SeNMFk. We demonstrate our new tool for literature review by applying it to two different fields in machine learning. Gradient descent is an essential optimization algorithm used to train neural networks. The curse of dimensionality affects performance in high-dimensional machine learning models. Identification of the family to which a malware specimen belongs is essential in understanding the behavior of the malware and developing mitigation strategies. Solutions proposed by prior work, however, are often not practicable due to the lack of realistic evaluation factors. These factors include learning under class imbalance, the ability to identify new malware, and the cost of production-quality labeled data. In practice, deployed models face prominent, rare, and new malware families. At the same time, obtaining a large quantity of up-to-date labeled malware for training a model can be expensive. In this paper, we address these problems and propose a novel hierarchical semi-supervised algorithm, which we call the HNMFk Classifier, that can be used in the early stages of the malware family labeling process. Our method is based on non-negative matrix factorization with automatic model selection, that is, with an estimation of the number of clusters. With HNMFk Classifier, we exploit the hierarchical structure of the malware data together with a semi-supervised setup, which enables us to classify malware families under conditions of extreme class imbalance. Our solution can perform abstaining predictions, or rejection option, which yields promising results in the identification of novel malware families and helps with maintaining the performance of the model when a low quantity of labeled data is used. We perform bulk classification of nearly 2,900 both rare and prominent malware families, through static analysis, using nearly 388,000 samples from the EMBER-2018 corpus. In our experiments, we surpass both supervised and semi-supervised baseline models with an F1 score of 0.80. Principal component analysis (PCA) helps in reducing the dimensionality of data while preserving variance.\n", + " score=0.9423 id=83913121-bc9c-4b12-ac93-5563ebd881a4 text=Reinforcement learning enables agents to learn optimal policies through trial and error. Highly specific datasets of scientific literature are important for both research and education. However, it is difficult to build such datasets at scale. A common approach is to build these datasets reductively by applying topic modeling on an established corpus and selecting specific topics. A more robust but time-consuming approach is to build the dataset constructively in which a subject matter expert (SME) handpicks documents. This method does not scale and is prone to error as the dataset grows. Here we showcase a new tool, based on machine learning, for constructively generating targeted datasets of scientific literature. Given a small initial “core” corpus of papers, we build a citation network of documents. At each step of the citation network, we generate text embeddings and visualize the embeddings through dimensionality reduction. Papers are kept in the dataset if they are “similar” to the core or are otherwise pruned through human-in-the-loop selection. Additional insight into the papers is gained through sub-topic modeling using SeNMFk. We demonstrate our new tool for literature review by applying it to two different fields in machine learning. Feature selection plays a crucial role in improving model efficiency and reducing overfitting. Ensemble methods such as bagging and boosting combine multiple models to improve predictions. Blockchain technology enhances security and transparency in AI-driven systems. Deep learning has revolutionized the field of computer vision and natural language processing. Blockchain technology enhances security and transparency in AI-driven systems. Convolutional neural networks excel at tasks such as image classification and object detection. Deep reinforcement learning has achieved breakthroughs in robotics and gaming AI.\n" + ] + } + ], + "source": [ + "# test_termite_e2e.py\n", + "import os, pandas as pd\n", + "from TELF.applications import Termite\n", + "\n", + "# make sure Termite picks OpenSearch\n", + "os.environ[\"EMBEDDING_STORE\"] = \"opensearch\"\n", + "os.environ[\"OS_HOST\"] = os.getenv(\"OS_HOST\", \"localhost\")\n", + "os.environ[\"OS_PORT\"] = os.getenv(\"OS_PORT\", \"9200\")\n", + "os.environ[\"OS_USE_SSL\"] = \"false\"\n", + "\n", + "\n", + "df = pd.read_csv(raw_csv_path)\n", + "\n", + "t = Termite(kg_credentials=None, verbose=True, model_name=\"malteos/scincl\")\n", + "\n", + "# compute embeddings (dict: row_index -> vector) and align with df order\n", + "emb_map = t.compute_embeddings(df, model_name=\"malteos/scincl\")\n", + "embeddings = [emb_map[i] for i in df.index]\n", + "dim = len(embeddings[0])\n", + "\n", + "index_name = \"termite_vectors_test_e2e\"\n", + "\n", + "# (re)create index with correct dim\n", + "t.store.ensure_index(index=index_name, dim=dim, metric=\"cosine\")\n", + "\n", + "# build payloads so 'text' is present at top level\n", + "ids = df[\"eid\"].astype(str).tolist()\n", + "payloads = [{\"text\": txt} for txt in df[\"abstract\"].astype(str).tolist()]\n", + "\n", + "# upsert (ids + vectors + payloads)\n", + "t.store.upsert(index_name, ids, embeddings, payloads=payloads)\n", + "\n", + "# --- add near the top ---\n", + "MODEL_NAME = \"malteos/scincl\" # keep consistent with your index\n", + "\n", + "def embed_text_with_termite(termite, text: str):\n", + " import pandas as pd\n", + " # Use the same column name your DF used (abstract)\n", + " qdf = pd.DataFrame({\"abstract\": [text]})\n", + " qmap = termite.compute_embeddings(qdf, model_name=MODEL_NAME)\n", + " return qmap[qdf.index[0]]\n", + "\n", + "# Custom query text\n", + "query_text = \"What problem in real-world malware labeling does the HNMFk Classifier aim to solve?\" # << your text here\n", + "\n", + "# 1) embed the query text\n", + "qvec = embed_text_with_termite(t, query_text)\n", + "\n", + "# 2) run kNN against your index\n", + "hits = t.store.search(index_name, qvec, k=5, source_fields=\"id,text\")\n", + "\n", + "print(\"\\nQuery:\", query_text)\n", + "print(\"Top-5 hits:\")\n", + "for _id, score, src in hits:\n", + " print(f\" score={score:.4f} id={_id} text={src.get('text')}\")\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "TELF", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/Termite/docker-compose-opensearch.yml b/examples/Termite/docker-compose-opensearch.yml new file mode 100644 index 00000000..3413da0e --- /dev/null +++ b/examples/Termite/docker-compose-opensearch.yml @@ -0,0 +1,86 @@ +name: telf-stack + +services: + # ------------------------- + # Neo4j (with APOC plugin) — unchanged + # ------------------------- + neo4j_termite: + image: neo4j:5.23 + container_name: neo4j_termite + restart: unless-stopped + ports: + - "7999:7474" # HTTP + - "7666:7687" # Bolt + environment: + NEO4J_AUTH: "neo4j/local_password" + NEO4J_server_default__listen__address: "0.0.0.0" + NEO4J_server_default__advertised__address: "localhost" + NEO4J_PLUGINS: '["apoc"]' + NEO4J_dbms_security_procedures_unrestricted: "apoc.*" + NEO4J_dbms_security_procedures_allowlist: "apoc.*" + NEO4J_dbms_security_auth__minimum__password__length: "12" + volumes: + - ./01_termite_output/DOCKERDATA/neo4j/data:/data + - ./01_termite_output/DOCKERDATA/neo4j/logs:/logs + - ./01_termite_output/DOCKERDATA/neo4j/import:/var/lib/neo4j/import + - ./01_termite_output/DOCKERDATA/neo4j/plugins:/plugins + networks: [telf-net] + + # ------------------------- + # OpenSearch (single-node with k-NN) + # ------------------------- + opensearch: + image: opensearchproject/opensearch:2.13.0 + container_name: opensearch + restart: unless-stopped + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=ChangeMe_1Strong! # <-- required + - discovery.type=single-node + - plugins.security.disabled=true # dev only + - bootstrap.memory_lock=true + - OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g # tune for your host + - cluster.name=telf-opensearch + - node.name=os-node-1 + - network.host=0.0.0.0 + # Optional: speed up k-NN warmup for small hosts + - knn.memory.circuit_breaker.enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - ./01_termite_output/DOCKERDATA/opensearch/data:/usr/share/opensearch/data + - ./01_termite_output/DOCKERDATA/opensearch/logs:/usr/share/opensearch/logs + ports: + - "9200:9200" # REST API + - "9600:9600" # Performance analyzer + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200 >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + networks: [telf-net] + + # ------------------------- + # OpenSearch Dashboards (optional UI) + # ------------------------- + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:2.13.0 + container_name: opensearch-dashboards + restart: unless-stopped + environment: + OPENSEARCH_HOSTS: '["http://opensearch:9200"]' + OPENSEARCH_SECURITY_DISABLED: "true" # dev only + SERVER_HOST: "0.0.0.0" + depends_on: + - opensearch + ports: + - "5601:5601" # visit http://localhost:5601 + networks: [telf-net] + +networks: + telf-net: + driver: bridge diff --git a/examples/Termite/docker-compose.yml b/examples/Termite/docker-compose.yml new file mode 100644 index 00000000..dfe60608 --- /dev/null +++ b/examples/Termite/docker-compose.yml @@ -0,0 +1,123 @@ +name: telf-stack + +services: + # ------------------------- + # Neo4j (with APOC plugin) + # ------------------------- + neo4j_termite: + image: neo4j:5.23 + container_name: neo4j_termite + restart: unless-stopped + ports: + - "7999:7474" # HTTP + - "7666:7687" # Bolt + environment: + # --- Security / auth --- + NEO4J_AUTH: "neo4j/local_password" + + # --- Networking (listen/advertised addresses) --- + NEO4J_server_default__listen__address: "0.0.0.0" + NEO4J_server_default__advertised__address: "localhost" + + # --- APOC plugin & procedure permissions --- + NEO4J_PLUGINS: '["apoc"]' + NEO4J_dbms_security_procedures_unrestricted: "apoc.*" + NEO4J_dbms_security_procedures_allowlist: "apoc.*" + + # Optional: stronger password policy + NEO4J_dbms_security_auth__minimum__password__length: "12" + volumes: + - ./01_termite_output/DOCKERDATA/neo4j/data:/data + - ./01_termite_output/DOCKERDATA/neo4j/logs:/logs + - ./01_termite_output/DOCKERDATA/neo4j/import:/var/lib/neo4j/import + - ./01_termite_output/DOCKERDATA/neo4j/plugins:/plugins + networks: [telf-net] + + # ------------------------- + # Milvus dependencies + # ------------------------- + milvus-etcd: + image: quay.io/coreos/etcd:v3.5.5 + container_name: milvus-etcd + restart: unless-stopped + environment: + ETCD_AUTO_COMPACTION_MODE: "revision" + ETCD_AUTO_COMPACTION_RETENTION: "1000" + ETCD_QUOTA_BACKEND_BYTES: "4294967296" + ETCD_SNAPSHOT_COUNT: "50000" + command: > + etcd -advertise-client-urls=http://0.0.0.0:2379 + -listen-client-urls=http://0.0.0.0:2379 + --data-dir=/etcd + volumes: + - ./01_termite_output/DOCKERDATA/milvus/etcd:/etcd + networks: [telf-net] + # port exposure optional; remove if you don't need host access + ports: + - "2379:2379" + + milvus-minio: + image: minio/minio:RELEASE.2024-01-13T07-53-03Z + container_name: milvus-minio + restart: unless-stopped + environment: + MINIO_ROOT_USER: "minioadmin" + MINIO_ROOT_PASSWORD: "minioadmin" + command: server /minio_data --console-address ":9001" + volumes: + - ./DOCKERDATA/milvus/minio:/minio_data + # MinIO image doesn't ship curl/wget; skip healthcheck to avoid false failures + networks: [telf-net] + ports: + - "9002:9000" # S3 API (host↔container) + - "9003:9001" # Console (host↔container) + # ------------------------- + # Milvus Standalone (2.4.x) + # ------------------------- + milvus-termite: + image: milvusdb/milvus:v2.4.23 + container_name: milvus-termite + restart: unless-stopped + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: "milvus-etcd:2379" + MINIO_ADDRESS: "milvus-minio:9000" + # If you enable auth in milvus.yaml later: + # MINIO_ACCESS_KEY: "minioadmin" + # MINIO_SECRET_KEY: "minioadmin" + depends_on: + - milvus-etcd + - milvus-minio + volumes: + - ./01_termite_output/DOCKERDATA/milvus/data:/var/lib/milvus + # To customize Milvus config later, copy it out and mount it: + # - ./DOCKERDATA/milvus/milvus.yaml:/milvus/configs/milvus.yaml + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9091/healthz || exit 1"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" # Milvus gRPC (pymilvus connects here) + - "9091:9091" # Milvus REST/gateway & web UI + networks: [telf-net] + + # ------------------------- + # (Optional) Attu – Milvus web UI + # ------------------------- + attu: + image: zilliz/attu:v2.5.12 + container_name: attu + restart: unless-stopped + environment: + MILVUS_URL: "milvus-termite:19530" + depends_on: + - milvus-termite + ports: + - "8000:3000" # Visit http://localhost:8000 + networks: [telf-net] + +networks: + telf-net: + driver: bridge diff --git a/post_install.py b/post_install.py index 18334f4b..d6bf1936 100644 --- a/post_install.py +++ b/post_install.py @@ -1,53 +1,201 @@ +#!/usr/bin/env python3 +""" +Post-install helper for TELF. + +- Installs (or verifies) spaCy + models and NLTK data in the *current* Python env. +- Optional GPU/HPC helpers via flags (uses conda consistently with -y). +- Avoids changing NumPy versions or bypassing your resolver. + +Usage examples: + python post_install.py + python post_install.py --gpu + python post_install.py --hpc-conda + python post_install.py --gpu --gpu-toolkit +""" + import argparse +import importlib.util import subprocess +import sys +from shutil import which + + +def run(cmd, **kw): + """Run a command with check=True and echo it.""" + print(">", " ".join(cmd)) + subprocess.run(cmd, check=True, **kw) + + +def has_module(mod_name: str) -> bool: + return importlib.util.find_spec(mod_name) is not None + + +def ensure_pkg(import_name: str, pip_name: str | None = None, version: str | None = None): + """ + Ensure `import_name` can be imported. If not, pip-install into THIS interpreter. + """ + try: + __import__(import_name) + except ModuleNotFoundError: + to_install = pip_name or import_name + if version: + to_install = f"{to_install}=={version}" + run([sys.executable, "-m", "pip", "install", to_install]) + + +def ensure_spacy_and_models(): + """ + Make sure spaCy is present and large/transformer models are available. + Model downloads are skipped if already installed. + """ + # Keep versions aligned with your pyproject (adjust if you bump there) + ensure_pkg("spacy", "spacy", "3.8.2") + + # Only install nltk if we’re actually going to download NLTK data later. + # We do it here so the import is guaranteed to work for the downloader. + ensure_pkg("nltk", "nltk", "3.9.1") + + # Install spaCy models only if missing + if not has_module("en_core_web_lg"): + run([sys.executable, "-m", "spacy", "download", "en_core_web_lg"]) + else: + print("spaCy model en_core_web_lg already present; skipping.") + + if not has_module("en_core_web_trf"): + run([sys.executable, "-m", "spacy", "download", "en_core_web_trf"]) + else: + print("spaCy model en_core_web_trf already present; skipping.") + + +def download_nltk_data(): + """ + Download NLTK corpora via API (more robust than `-m nltk.downloader`). + Skips re-downloads. + """ + import nltk + + for pkg in ("wordnet", "omw-1.4"): + print(f"Ensuring NLTK data: {pkg}") + nltk.download(pkg, quiet=True) + + +def conda_required(): + if which("conda") is None: + raise RuntimeError( + "Conda is required for the requested GPU/HPC (conda) operations, " + "but 'conda' was not found on PATH." + ) + + +def conda_install(*packages: str, channel: str = "conda-forge"): + """ + Install conda packages non-interactively from a consistent channel. + """ + conda_required() + run(["conda", "install", "-y", "-c", channel, *packages]) + + +def install_gpu_dependencies(via_conda_toolkit: bool, install_cupy: bool): + """ + Optionally install CUDA toolkit pieces and CuPy. Uses conda-forge consistently. + """ + if via_conda_toolkit: + print("Installing CUDA toolkit components (conda-forge)...") + conda_install("cudatoolkit", channel="conda-forge") + conda_install("cudnn", channel="conda-forge") + + if install_cupy: + print("Installing CuPy (conda-forge)...") + conda_install("cupy", channel="conda-forge") + + +def install_mpi(hpc_pip: bool, hpc_conda: bool): + """ + Optionally install mpi4py either via pip (requires system MPI toolchain) + or via conda-forge (preferred for portability). + """ + if hpc_pip and hpc_conda: + # Prefer conda-forge to avoid system MPI mismatches + print("Both --hpc and --hpc-conda were set; preferring conda-forge build.") + hpc_pip = False -def run_post_install_commands(gpu=False, hpc=False, hpc_conda=False, gpu_toolkit=False): - - if hpc and hpc_conda: - print("Both HPC pip and HPC conda were True. Defaulting to hpc via pip.") - hpc_conda = False - - print("Downloading SpaCy en_core_web_lg model...") - subprocess.run(["python", "-m", "spacy", "download", "en_core_web_lg"]) - print("Downloading SpaCy en_core_web_trf model...") - subprocess.run(["python", "-m", "spacy", "download", "en_core_web_trf"]) - print("Downloading NLTK wordnet and omw-1.4 data...") - subprocess.run(["python", "-m", "nltk.downloader", "wordnet", "omw-1.4"]) - - if gpu: - print("Installing Cupy...") - subprocess.run(["conda", "install", "-c", "conda-forge", "cupy", "numpy==2.0.0"]) - if hpc: - print("Installing mpi4py via pip") - subprocess.run(["pip", "install", "mpi4py"]) if hpc_conda: - print("Installing mpi4py via conda-forge") - subprocess.run(["conda", "install", "-c", "conda-forge", "mpi4py"]) - if gpu_toolkit: - print("Installing cudnn and cudatoolkit") - subprocess.run(["conda", "install", "cudatoolkit"]) - subprocess.run(["conda", "install", "cudnn"]) + print("Installing mpi4py via conda-forge...") + conda_install("mpi4py", channel="conda-forge") + elif hpc_pip: + print("Installing mpi4py via pip (requires system MPI headers/libs)...") + run([sys.executable, "-m", "pip", "install", "mpi4py"]) + + +def run_post_install_commands( + gpu: bool = False, + hpc: bool = False, + hpc_conda: bool = False, + gpu_toolkit: bool = False, + skip_models: bool = False, +): + """ + Execute post-install steps in a safe, idempotent way. + """ + # 1) NLP bits (spaCy + models, NLTK data) + if not skip_models: + ensure_spacy_and_models() + download_nltk_data() + else: + print("Skipping spaCy model and NLTK data steps (--skip-models).") + + # 2) GPU deps (optional) + if gpu_toolkit or gpu: + install_gpu_dependencies(via_conda_toolkit=gpu_toolkit, install_cupy=gpu) + + # 3) HPC MPI (optional) + install_mpi(hpc_pip=hpc, hpc_conda=hpc_conda) + + print("Post-install completed successfully.") + + +def main(): + p = argparse.ArgumentParser( + description="Post installation script for downloading models/data and optional GPU/HPC extras." + ) + p.add_argument("--gpu", action="store_true", help="Install CuPy via conda-forge.") + p.add_argument( + "--gpu-toolkit", + action="store_true", + help="Install cudatoolkit and cudnn via conda-forge.", + ) + p.add_argument( + "--hpc", + action="store_true", + help="Install mpi4py via pip (requires compatible system MPI).", + ) + p.add_argument( + "--hpc-conda", + action="store_true", + help="Install mpi4py via conda-forge (preferred for portability).", + ) + p.add_argument( + "--skip-models", + action="store_true", + help="Skip spaCy model downloads and NLTK data steps.", + ) + + args = p.parse_args() + try: + run_post_install_commands( + gpu=args.gpu, + hpc=args.hpc, + hpc_conda=args.hpc_conda, + gpu_toolkit=args.gpu_toolkit, + skip_models=args.skip_models, + ) + except subprocess.CalledProcessError as e: + print(f"\nCommand failed with exit code {e.returncode}:\n {' '.join(e.cmd)}") + sys.exit(e.returncode) + except RuntimeError as e: + print(f"\nERROR: {e}") + sys.exit(1) - # correct the numpy version - subprocess.run(["pip", "install", "numpy==2.0"]) if __name__ == "__main__": - # Create argument parser - parser = argparse.ArgumentParser(description="Post installation script for downloading models and data.") - - # Add arguments to control whether specific downloads happen - parser.add_argument('--gpu', action='store_true', help="Install Cupy if using GPU") - parser.add_argument('--hpc', action='store_true', help="Install mpi4py if using HPC using pip") - parser.add_argument('--hpc-conda', action='store_true', help="Install mpi4py if using HPC using conda-forge") - parser.add_argument('--gpu-toolkit', action='store_true', help="Install cudatoolkit and cudnn that may be needed in some systems.") - - # Parse the arguments - args = parser.parse_args() - - # Call the function with parsed arguments - run_post_install_commands( - gpu=args.gpu, - hpc=args.hpc, - hpc_conda=args.hpc_conda, - gpu_toolkit=args.gpu_toolkit - ) \ No newline at end of file + main() diff --git a/pyproject.toml b/pyproject.toml index 436c66db..5b91930d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ packages = [ [tool.poetry.dependencies] python = "3.11.10" -numpy = "<=2.0.0" +numpy = "1.26.4" scipy = "^1.14.1" matplotlib = "^3.9.2" pathos = "^0.3.3" @@ -41,8 +41,8 @@ rbloom = "^1.5.2" pymongo = "^4.11.2" seaborn = "^0.13.2" umap-learn = "^0.5.7" -torch = "^2.6.0" -transformers = "^4.39.3" +torch = ">=2.2,<2.3" +transformers = ">=4.39,<5" langchain-ollama = "^0.3.0" openai = "^1.69.0" pyvis = "^0.3.2" @@ -54,7 +54,13 @@ watchdog = "^6.0.0" pillow = "^11.1.0" pykeen = "^1.11.0" pycountry = "^24.6.1" +neo4j = "^5.26.0" +pymilvus = "2.4.0" +jupyterlab = "^4.2.5" +notebook = "^7.2.2" +ipywidgets = "^8.1.5" +opensearch-py = "^3.0.0" [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +build-backend = "poetry.core.masonry.api" \ No newline at end of file From 56e7241f7f142985e4813948ff6f45cb88d4e509 Mon Sep 17 00:00:00 2001 From: Ryan Calvin Barron Date: Wed, 24 Sep 2025 17:35:27 -0600 Subject: [PATCH 02/13] peacock block debug, iteration enumeration in block operation which includes renumbering bloks on run if they change, and post install script update for peacock compatibility --- README.md | 8 +- .../pages/helpers/load_project_data.py | 219 ++- TELF/applications/Termite/misc/material_kg.py | 45 + TELF/helpers/figures.py | 21 + TELF/pipeline/__init__.py | 4 + TELF/pipeline/block_manager.py | 311 ++- TELF/pipeline/blocks/__init__.py | 5 +- TELF/pipeline/blocks/artic_fox_block.py | 49 +- TELF/pipeline/blocks/base_block.py | 126 +- .../blocks/collect_hnmfk_leaf_block.py | 447 +++++ TELF/pipeline/blocks/peacock_stats_block.py | 8 +- TELF/pipeline/blocks/termite_neo4j_block.py | 417 ++++ TELF/pipeline/blocks/termite_vector_block.py | 137 ++ TELF/pipeline/blocks/wolf_block.py | 3 + .../post_process_example.ipynb | 186 +- ...mantic_hnmfk_collection_slurm_option.ipynb | 1730 +++++++++++++++++ post_install/__init__.py | 1 + post_install/__main__.py | 335 ++++ pyproject.toml | 13 +- 19 files changed, 3913 insertions(+), 152 deletions(-) create mode 100644 TELF/applications/Termite/misc/material_kg.py create mode 100644 TELF/pipeline/blocks/collect_hnmfk_leaf_block.py create mode 100644 TELF/pipeline/blocks/termite_neo4j_block.py create mode 100644 TELF/pipeline/blocks/termite_vector_block.py create mode 100644 examples/Full TELF Pipeline/single_block_examples/semantic_hnmfk_collection_slurm_option.ipynb create mode 100644 post_install/__init__.py create mode 100644 post_install/__main__.py diff --git a/README.md b/README.md index 5d8ce9e9..f58f6241 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,11 @@ conda develop . Next, we need to install the optional and additional dependencies. These include optional dependencies for GPU and HPC capabilities, as well as required dependencies like the SpaCy language models. To view all available options, please run: ```shell -python post_install.py --help +telf-post-install --help ``` Install the additional dependencies: ```shell -python post_install.py # use the following, for example, for GPU system: +telf-post-install # use the following, for example, for GPU system: < telf-post-install --gpu> ``` #### Jupyter Setup Tutorial for using the examples ([Link](https://www.maksimeren.com/post/conda-and-jupyter-setup-for-research/)) @@ -228,7 +228,7 @@ Developer test suites are located under [```tests/```](tests/) directory. Tests conda create --prefix= python=3.11.10 source activate # or use conda activate <...> pip install . -python post_install.py --gpu --hpc-conda +telf-post-install --gpu --hpc-conda ``` ### Darwin @@ -239,5 +239,5 @@ module load miniconda3 conda create --name TELF python=3.11.10 conda activate TELF # or pip install . -python post_install.py --gpu --hpc +telf-post-install --gpu --hpc ``` \ No newline at end of file diff --git a/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py b/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py index a39f7643..62ddab11 100644 --- a/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py +++ b/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py @@ -9,6 +9,8 @@ import torch import pickle import json +import csv +from collections import defaultdict @st.cache_resource def load_link_data(path): @@ -181,36 +183,213 @@ def add_children(node_name): tree.append(add_children(node["name"])) return tree -def parse_topic_folder_name(folder_name, path=None): +DEBUG = False # prints to terminal + +def _debug(msg): + if DEBUG: + print(msg, flush=True) + +# Matches cluster_for_k=N*.csv (case-insensitive, with optional extra suffix) +_K_FILE_PAT = re.compile(r'^cluster_for_k=(\d+)(?:[^/]*)\.csv$', re.IGNORECASE) + +# Cache per root path +_COUNTS_CACHE = {} +_COUNTS_SRC_CACHE = {} + +def _list_topic_dirs(root_path: str): + try: + dirs = [ + d for d in os.listdir(root_path) + if os.path.isdir(os.path.join(root_path, d)) + ] + # Heuristic: folders that start with digits are topic dirs (e.g., "3-foo", "12", etc.) + topic_dirs = [d for d in dirs if re.match(r'^\d+\b', d)] + return sorted(topic_dirs) + except FileNotFoundError: + return [] + +def _iter_candidate_csvs(root_path: str): + """Yield (abs_path, k) for cluster_for_k=*.csv at root and one level below.""" + if not root_path or not os.path.isdir(root_path): + _debug(f"[scan] Not a dir: {root_path}") + return + + _debug(f"[scan] Looking for cluster_for_k=*.csv under: {root_path}") + + # Top-level files + try: + for entry in os.scandir(root_path): + if entry.is_file(): + m = _K_FILE_PAT.match(entry.name) + if m: + _debug(f"[scan] Found CSV: {entry.name} (k={m.group(1)})") + yield entry.path, int(m.group(1)) + except FileNotFoundError: + pass + + # One level down (subfolders only) + try: + for entry in os.scandir(root_path): + if entry.is_dir(): + for sub in os.scandir(entry.path): + if sub.is_file(): + m = _K_FILE_PAT.match(sub.name) + if m: + _debug(f"[scan] Found CSV (subdir): {entry.name}/{sub.name} (k={m.group(1)})") + yield sub.path, int(m.group(1)) + except FileNotFoundError: + pass + +def _norm_cluster_key(val): + """Normalize cluster key to a stringified integer if possible (e.g., '3', 3, '3.0', 'cluster_3').""" + if val is None: + return None + if isinstance(val, (int, float)): + try: + return str(int(val)) + except Exception: + return str(val).strip() + s = str(val).strip() + m = re.search(r'(\d+)', s) + return m.group(1) if m else s # fallback to raw string + +def _read_counts_map_from_csv(csv_path: str): + """Build counts map {cluster_id(str): count(int)} using the 'cluster' column (case-insensitive).""" + _debug(f"[counts] Reading CSV: {csv_path}") + counts = defaultdict(int) + try: + with open(csv_path, "r", newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + if not reader.fieldnames: + _debug("[counts] No header/fieldnames found.") + return {} + + # find 'cluster' column case-insensitively + cluster_col = None + for name in reader.fieldnames: + if name and name.strip().lower() == "cluster": + cluster_col = name + break + + if not cluster_col: + _debug(f"[counts] 'cluster' column not found in {reader.fieldnames}") + return {} + + row_total = 0 + for row in reader: + row_total += 1 + key = _norm_cluster_key(row.get(cluster_col)) + if key is not None and key != "": + counts[key] += 1 + + _debug(f"[counts] Total rows read: {row_total}. Unique clusters: {len(counts)}") + except Exception as e: + _debug(f"[counts] Error reading {csv_path}: {e}") + return {} + return dict(counts) + +def _choose_best_cluster_csv(root_path: str): + """Choose the best cluster_for_k=*.csv. Prefer k == number of topic dirs; else largest k.""" + candidates = list(_iter_candidate_csvs(root_path)) + if not candidates: + _debug("[choose] No cluster_for_k=*.csv candidates found.") + return None + + topic_dirs = _list_topic_dirs(root_path) + k_target = len(topic_dirs) + _debug(f"[choose] Topic dirs detected: {k_target}") + + # Prefer exact match to number of topic dirs + exact = [p for p in candidates if p[1] == k_target and k_target > 0] + if exact: + # If multiple with same k, pick the shortest path (heuristic) + chosen = sorted(exact, key=lambda x: (len(x[0]), x[0]))[0] + _debug(f"[choose] Using exact k match: {os.path.basename(chosen[0])} (k={chosen[1]})") + return chosen[0] + + # Else pick largest k + candidates.sort(key=lambda x: x[1], reverse=True) + chosen = candidates[0] + _debug(f"[choose] Using largest k: {os.path.basename(chosen[0])} (k={chosen[1]})") + return chosen[0] + +def _get_counts_map(root_path: str): + """Get (and cache) the counts map for this root path.""" + if root_path in _COUNTS_CACHE: + return _COUNTS_CACHE[root_path], _COUNTS_SRC_CACHE.get(root_path) + + csv_path = _choose_best_cluster_csv(root_path) + if not csv_path: + _COUNTS_CACHE[root_path] = {} + _COUNTS_SRC_CACHE[root_path] = None + return {}, None + + counts_map = _read_counts_map_from_csv(csv_path) + _COUNTS_CACHE[root_path] = counts_map + _COUNTS_SRC_CACHE[root_path] = csv_path + return counts_map, csv_path + +def parse_topic_folder_name(folder_name: str, path: str | None = None): """ Extracts topic number, label, and document count from folder name. - - Expected format: topic_number-label_with_underscores-documents_count-documents - Example: '3-label_of_the_topic_9-documents' -> ('3', 'Label of the topic', '9') - - Args: - folder_name (str): Folder name formatted as: -_-documents - - Returns: - tuple: (str, str, str) -> (topic_number, cleaned_label, document_count) - or (None, None, None) if parsing fails. + If the count isn't in the name, read it from cluster_for_k=N.csv at the root (covers all clusters), + using the 'cluster' column to count rows per cluster ID. """ - match = re.match(r'(\d+)-([^-]+)_(\d+)-documents', folder_name) + _debug(f"\n[parse] Folder: {folder_name}") + match = re.match(r'^(\d+)-([^-]+?)(?:[_-](\d+)-documents)?$', folder_name) if match: - topic_number = match.group(1) # Extract topic number - label = match.group(2).replace("_", " ").strip() # Replace underscores with spaces - document_count = match.group(3) # Extract document count + topic_number = match.group(1) + label = match.group(2).replace("_", " ").strip() + document_count = match.group(3) + _debug(f"[parse] Parsed -> number={topic_number}, label='{label}', name_count={document_count}") + + if document_count is None and path: + counts_map, csv_src = _get_counts_map(path) + key = _norm_cluster_key(topic_number) + inferred = counts_map.get(key) + if inferred is not None: + document_count = str(inferred) + _debug(f"[parse] Inferred from CSV ({os.path.basename(csv_src) if csv_src else 'n/a'}): {document_count}") + else: + _debug(f"[parse] No count for cluster={key} in CSV.") + document_count = "Unknown" + + if document_count is None: + document_count = "Unknown" + + _debug(f"[parse] Result -> ({topic_number}, '{label}', {document_count})") return topic_number, label, document_count + elif folder_name.isdigit(): - labels = load_csv_file_items(path, suffix="cluster_summaries", ends=False, column="label") - if path is None or labels is None: - return folder_name, "Topic", "Unknown" - else: - return folder_name, labels[int(folder_name)], "Unknown" + topic_number = folder_name + # Try label mapping (your existing helper) + label = "Topic" + try: + labels = load_csv_file_items(path, suffix="cluster_summaries", ends=False, column="label") + if path is not None and labels is not None: + label = labels[int(folder_name)] + except Exception as e: + _debug(f"[label] Could not map label: {e}") + + document_count = "Unknown" + if path: + counts_map, csv_src = _get_counts_map(path) + key = _norm_cluster_key(topic_number) + inferred = counts_map.get(key) + if inferred is not None: + document_count = str(inferred) + _debug(f"[parse] Inferred (numeric folder) from CSV ({os.path.basename(csv_src) if csv_src else 'n/a'}): {document_count}") + else: + _debug(f"[parse] No count for cluster={key} in CSV (numeric folder).") + + _debug(f"[parse] Numeric folder -> ({topic_number}, '{label}', {document_count})") + return topic_number, label, document_count + _debug("[parse] Unrecognized folder name format.") return None, None, None + def map_folder_to_logical_name(folder_name: str, root_name) -> str: """ Convert a physical folder name (like '*_0', '*_1_2', or '0_1') diff --git a/TELF/applications/Termite/misc/material_kg.py b/TELF/applications/Termite/misc/material_kg.py new file mode 100644 index 00000000..343fd18a --- /dev/null +++ b/TELF/applications/Termite/misc/material_kg.py @@ -0,0 +1,45 @@ +# combined_material_clusters_path = "./data/material_injection.csv" +# combined_material_clusters_path = "./affiliation_resolve_materials.csv" +from TELF.applications import Termite +from TELF.applications.Termite.neo4j_termite import * +termite = Termite() +aflow_path = "./material_properties.csv" +# mats_and_NER_path = "./materials_and_MATNER_triplets.csv" +aflow_trip_path = "./aflow_triplets.csv" + +ICSD_TYPE = 'ICSD' +CRYSTAL_SYSTEM_TYPE = 'Crystal_system' +CRYSTAL_CLASS_TYPE ='Crystal_class' +PEARSON_TYPE = 'Pearson' +SPACEGROUP_TYPE = 'Spacegroup' + +MATERIAL_ICSD_RELATION = 'crystal_database_id' +ICSD_SYSTEM_RELATION = 'id_is_system' +ICSD_CLASS_RELATION ='id_is_class' +ICSD_PEARSON_RELATION ='id_is_pearson' +ICSD_SPACEGROUP_RELATION ='id_is_spacegroup' + +material_triplet_map = { + 'ENTITIES': + [ + {ET:MATERIAL_TYPE, FROM_COL: 'material', MAKE_ID_UNIQUE:True}, + {ET:ICSD_TYPE, FROM_COL: 'ICSD', MAKE_ID_UNIQUE:True,ATTR_COL:[{FROM_COL: 'geometry', ATTR_NAME:'geometry'}, {FROM_COL: 'Egap', ATTR_NAME:'Egap'} ]}, + {ET:CRYSTAL_SYSTEM_TYPE, FROM_COL: 'crystal_system', MAKE_ID_UNIQUE:True}, + {ET:CRYSTAL_CLASS_TYPE, FROM_COL: 'crystal_class', MAKE_ID_UNIQUE:True}, + {ET:PEARSON_TYPE, FROM_COL: 'pearson', MAKE_ID_UNIQUE:True}, + {ET:SPACEGROUP_TYPE, FROM_COL: 'spacegroup', MAKE_ID_UNIQUE:True}, + + ], + 'RELATIONS': + [ + {HT:MATERIAL_TYPE, R:MATERIAL_ICSD_RELATION, TT:ICSD_TYPE}, + {HT:ICSD_TYPE, R:ICSD_SYSTEM_RELATION , TT: CRYSTAL_SYSTEM_TYPE, }, + {HT:ICSD_TYPE, R:ICSD_CLASS_RELATION , TT:CRYSTAL_CLASS_TYPE , }, + {HT:ICSD_TYPE, R: ICSD_PEARSON_RELATION, TT: PEARSON_TYPE, }, + {HT:ICSD_TYPE, R: ICSD_SPACEGROUP_RELATION, TT: SPACEGROUP_TYPE, }, + + ] +} +termite.make_unique_constrains( material_triplet_map) +termite.from_csv_to_triplets(aflow_path, aflow_trip_path, material_triplet_map) +termite.update_database_multithreaded(aflow_trip_path,start_from=0,shuffle_rows=True ) \ No newline at end of file diff --git a/TELF/helpers/figures.py b/TELF/helpers/figures.py index 671c42f9..75cb5334 100644 --- a/TELF/helpers/figures.py +++ b/TELF/helpers/figures.py @@ -27,6 +27,27 @@ def plot_authors_graph(df, id_col='s2_author_ids', name_col='s2_authors', title='Co-Authors Graph', width=900, height=900, max_node_size=50, min_node_size=3): G = create_authors_graph(df, id_col) + + # -------------------------- bail out gracefully on empty graphs -------------------------- + if G.number_of_nodes() == 0 or G.number_of_edges() == 0: + fig = go.Figure() + fig.update_layout( + title=f"{title} — no data to display", + width=width, + height=height, + xaxis=dict(visible=False), + yaxis=dict(visible=False), + showlegend=False, + margin=dict(l=20, r=20, t=60, b=20), + ) + # Optional: a centered annotation so the HTML isn't just a blank box + fig.add_annotation( + text="No relationships found for the given data/columns.", + x=0.5, y=0.5, xref="paper", yref="paper", + showarrow=False + ) + return fig + pos = nx.spring_layout(G) # position nodes using networkx's spring layout name_map = get_id_to_name(df, name_col, id_col) diff --git a/TELF/pipeline/__init__.py b/TELF/pipeline/__init__.py index 94f7ca8a..1da1e48e 100644 --- a/TELF/pipeline/__init__.py +++ b/TELF/pipeline/__init__.py @@ -67,3 +67,7 @@ from .blocks.term_table_block import TermTableBlock from .blocks.spacey_NER_block import SpacyNERBlock + +from .blocks.collect_hnmfk_leaf_block import CollectHNMFkLeafBlock +from .blocks.termite_neo4j_block import TermiteNeo4jBlock +from .blocks.termite_vector_block import TermiteVectorBlock \ No newline at end of file diff --git a/TELF/pipeline/block_manager.py b/TELF/pipeline/block_manager.py index 8eb86d25..6c381274 100644 --- a/TELF/pipeline/block_manager.py +++ b/TELF/pipeline/block_manager.py @@ -57,6 +57,48 @@ def __init__( self.describe_io() def __call__(self) -> DataBundle: + base = Path(self.bundle[SAVE_DIR_BUNDLE_KEY]) + + # 1) Preflight: rename block directories and update checkpoint paths on disk + # MUST return: + # - base_to_display: {"SemanticHNMFk": "06_SemanticHNMFk", ...} + # - prefix_map: {"/.../07_SemanticHNMFk": "/.../06_SemanticHNMFk", ...} + base_to_display, prefix_map = _renumber_dirs_and_update_ckpts(base, self.blocks) + + # 2) Update in-memory objects to new prefixes BEFORE any block runs + # 2a) Rewrite any stored paths inside the bundle values + try: + for base_key, bucket in list(self.bundle._store.items()): + for tag, val in list(bucket.items()): + if tag == "_latest": + continue + # Apply all variants + new_val = val + for old_pref, new_pref in prefix_map.items(): + new_val = _deep_replace_in_obj(new_val, {old_pref: new_pref}) + bucket[tag] = new_val + except Exception: + pass + + # Set display tags and rewrite block settings (init/call) in memory + for block in self.blocks: + bt = _base_tag(getattr(block, "_original_tag", block.tag)) + if not hasattr(block, "_original_tag"): + block._original_tag = bt + block.tag = base_to_display.get(bt, bt) + + if isinstance(getattr(block, "init_settings", None), dict): + new_init = block.init_settings + for old_pref, new_pref in prefix_map.items(): + new_init = _deep_replace_in_obj(new_init, {old_pref: new_pref}) + block.init_settings = new_init + + if isinstance(getattr(block, "call_settings", None), dict): + new_call = block.call_settings + for old_pref, new_pref in prefix_map.items(): + new_call = _deep_replace_in_obj(new_call, {old_pref: new_pref}) + block.call_settings = new_call + total = len(self.blocks) log_dir: Path | None = None progress_fp = None @@ -69,6 +111,7 @@ def __call__(self) -> DataBundle: progress_fp.write("# IO table\n" + "\n".join(table_lines) + "\n\n") progress_fp.flush() + # 3) Run blocks for idx, block in enumerate(self.blocks, 1): # Override the block’s load_checkpoint flag if requested if self.force_checkpoint is not None: @@ -89,7 +132,6 @@ def __call__(self) -> DataBundle: buf_err.write(f"⚠️ Exception in block {block.tag}:\n") traceback.print_exc(file=buf_err) captured = buf_out.getvalue() + buf_err.getvalue() - # Write error log and re-raise ts = _dt.datetime.now().strftime("%Y%m%d_%H%M%S") (log_dir / f"{idx:02d}_{block.tag}_{ts}.log").write_text(captured, encoding="utf-8") raise @@ -108,6 +150,16 @@ def __call__(self) -> DataBundle: traceback.print_exc() raise + # 4) Alias outputs: also store under the base tag (no NN_ prefix) + disp_tag = block.tag # e.g., "06_SemanticHNMFk" + base_tag = _base_tag(disp_tag) # e.g., "SemanticHNMFk" + for base_key in self.bundle.keys_by_tag(disp_tag): + try: + val = self.bundle[f"{disp_tag}.{base_key}"] + self.bundle[f"{base_tag}.{base_key}"] = val + except KeyError: + pass + elapsed = time.perf_counter() - t0 if self.progress: print(f"✓ [{idx}/{total}] {block.tag} finished in {elapsed:,.2f}s") @@ -121,6 +173,7 @@ def __call__(self) -> DataBundle: progress_fp.close() return self.bundle + # ------------------------------------------------------------------ # # helper – produce describe-io lines without printing # # ------------------------------------------------------------------ # @@ -402,3 +455,259 @@ def _ensure_result_path(self) -> None: """ if "result_path" not in self.bundle: self.bundle["result_path"] = Path.cwd() / "results" + + + +import json, os, re, uuid, pickle +from pathlib import Path +from typing import Any + +INDEXED_DIR_RE = re.compile(r"^\d+_") +TEXT_EXTS = {".json", ".txt", ".yaml", ".yml", ".ini", ".cfg", ".csv", ".tsv"} +PICKLE_EXTS = {".p", ".pkl", ".pickle"} + +def _base_tag(name: str) -> str: + return INDEXED_DIR_RE.sub("", name) + +def _display_name(tag: str, idx: int, width: int) -> str: + return f"{idx:0{width}d}_{tag}" + +def _find_existing_dir(base: Path, tag: str) -> Path | None: + candidates = sorted( + base.glob(f"[0-9][0-9]*_{tag}"), + key=lambda p: p.stat().st_mtime if p.exists() else 0, + reverse=True, + ) + if candidates: + return candidates[0] + plain = base / tag + return plain if plain.exists() else None + +def _collect_ckpt_files(base: Path) -> list[Path]: + return list(base.rglob("__checkpoints__.json")) + +def _rewrite_ckpt_paths(ckpt_file: Path, prefix_map: dict[str, str]) -> bool: + try: + data = json.loads(ckpt_file.read_text(encoding="utf-8")) + except Exception: + return False + changed = False + for k, v in list(data.items()): + if isinstance(v, str): + for old_prefix, new_prefix in prefix_map.items(): + if v == old_prefix or v.startswith(old_prefix + os.sep): + data[k] = new_prefix + v[len(old_prefix):] + changed = True + if changed: + ckpt_file.write_text(json.dumps(data, indent=2), encoding="utf-8") + return changed + +def _deep_replace(obj: Any, old_prefix: str, new_prefix: str, seen: set[int] | None = None) -> tuple[bool, Any]: + if seen is None: + seen = set() + oid = id(obj) + if oid in seen: + return False, obj + seen.add(oid) + + if isinstance(obj, str): + if old_prefix in obj: + return True, obj.replace(old_prefix, new_prefix) + return False, obj + + if isinstance(obj, dict): + changed = False + out = {} + for k, v in obj.items(): + ck, nk = _deep_replace(k, old_prefix, new_prefix, seen) if isinstance(k, str) else (False, k) + cv, nv = _deep_replace(v, old_prefix, new_prefix, seen) + changed = changed or ck or cv + out[nk] = nv + return changed, out + + if isinstance(obj, list): + changed = False + out = [] + for v in obj: + cv, nv = _deep_replace(v, old_prefix, new_prefix, seen) + changed = changed or cv + out.append(nv) + return changed, out + + if isinstance(obj, tuple): + changed = False + out_list = [] + for v in obj: + cv, nv = _deep_replace(v, old_prefix, new_prefix, seen) + changed = changed or cv + out_list.append(nv) + return changed, tuple(out_list) + + try: + attrs = vars(obj) + except Exception: + return False, obj + + changed = False + for k, v in list(attrs.items()): + cv, nv = _deep_replace(v, old_prefix, new_prefix, seen) + if cv: + try: + setattr(obj, k, nv) + changed = True + except Exception: + pass + return changed, obj + +def _rewrite_internal_paths_in_tree(root: Path, prefix_map: dict[str, str]) -> None: + """ + Walk files under `root` and replace occurrences of *any* old→new prefix. + Handles text and pickle files; best-effort, silent on failures. + """ + for p in root.rglob("*"): + if not p.is_file(): + continue + ext = p.suffix.lower() + + if ext in TEXT_EXTS: + try: + s = p.read_text(encoding="utf-8", errors="ignore") + changed = False + for old_prefix, new_prefix in prefix_map.items(): + if old_prefix in s: + s = s.replace(old_prefix, new_prefix) + changed = True + if changed: + p.write_text(s, encoding="utf-8") + except Exception: + pass + continue + + if ext in PICKLE_EXTS: + try: + with p.open("rb") as f: + obj = pickle.load(f) + changed_any = False + for old_prefix, new_prefix in prefix_map.items(): + changed, obj = _deep_replace(obj, old_prefix, new_prefix) + changed_any = changed_any or changed + if changed_any: + with p.open("wb") as f: + pickle.dump(obj, f) + except Exception: + pass + continue + +def _renumber_dirs_and_update_ckpts(base: Path, blocks: list) -> tuple[dict[str, str], dict[str, str]]: + """ + Rename block directories to NN_, rewrite checkpoint JSONs, + and migrate internal absolute/relative paths inside files under renamed dirs. + RETURNS: + (base_to_display, prefix_map) where prefix_map includes abs+rel variants. + """ + base.mkdir(parents=True, exist_ok=True) + + width = max(2, len(str(len(blocks)))) + desired: list[tuple[str, str]] = [] + for i, block in enumerate(blocks, start=1): + bt = _base_tag(getattr(block, "_original_tag", block.tag)) + desired.append((bt, _display_name(bt, i, width))) + + # Plan renames + rename_plan: list[tuple[Path, Path]] = [] + for (bt, new_disp) in desired: + src = _find_existing_dir(base, bt) + if src is None: + continue + dst = base / new_disp + if src.resolve() != dst.resolve(): + rename_plan.append((src, dst)) + + # Two-phase rename to avoid collisions + temp_map: dict[Path, Path] = {} + for src, dst in rename_plan: + if not src.exists(): + continue + tmp = base / (src.name + f".tmp-{uuid.uuid4().hex[:8]}") + src.rename(tmp) + temp_map[tmp] = dst + + # Move temps to final + for tmp, dst in temp_map.items(): + if dst.exists(): + if dst.is_dir() and tmp.is_dir(): + for item in tmp.iterdir(): + target = dst / item.name + if not target.exists(): + item.rename(target) + tmp.rmdir() + else: + n = 1 + alt = Path(str(dst) + f".old{n}") + while alt.exists(): + n += 1 + alt = Path(str(dst) + f".old{n}") + tmp.rename(alt) + else: + tmp.rename(dst) + + # Build a rich prefix map (absolute + relative variants) + prefix_map = _build_prefix_map_variants(base, rename_plan) + + # Rewrite checkpoint JSON files using all variants + for ckpt in _collect_ckpt_files(base): + _rewrite_ckpt_paths(ckpt, prefix_map) + + # Rewrite internals inside each renamed directory tree (text+pickle) using all variants + for src, dst in rename_plan: + try: + _rewrite_internal_paths_in_tree(dst, prefix_map) + except Exception: + pass + + base_to_display = {bt: new_disp for (bt, new_disp) in desired} + return base_to_display, prefix_map + + +def _deep_replace_in_obj(obj, prefix_map: dict[str, str]): + from pathlib import Path as _Path + if isinstance(obj, (str, _Path)): + s = str(obj) + for old, new in prefix_map.items(): + if s == old or s.startswith(old + os.sep): + s = new + s[len(old):] + return _Path(s) if isinstance(obj, _Path) else s + if isinstance(obj, dict): + return { _deep_replace_in_obj(k, prefix_map) if isinstance(k, (str, _Path)) else k: + _deep_replace_in_obj(v, prefix_map) for k, v in obj.items() } + if isinstance(obj, list): + return [ _deep_replace_in_obj(v, prefix_map) for v in obj ] + if isinstance(obj, tuple): + return tuple(_deep_replace_in_obj(v, prefix_map) for v in obj) + return obj + +def _build_prefix_map_variants(base: Path, rename_plan: list[tuple[Path, Path]]) -> dict[str, str]: + """ + For each (src,dst) directory rename, return a mapping that includes: + - absolute: /abs/.../07_Tag -> /abs/.../06_Tag + - relative (from CWD): src -> dst (e.g., 'example_results/.../07_Tag' -> '.../06_Tag') + This catches both absolute and relative paths embedded in files or memory. + """ + m: dict[str, str] = {} + for src, dst in rename_plan: + # absolute variants + abs_old = str(src.resolve()) + abs_new = str(dst.resolve()) + m[abs_old] = abs_new + + # relative variants (as written on disk; your code uses Path(...) directly) + rel_old = str(src) # typically 'example_results/.../07_Tag' + rel_new = str(dst) + m[rel_old] = rel_new + + # Sometimes code stores a trailing slash; add those too + if not abs_old.endswith(os.sep): + m[abs_old + os.sep] = abs_new + os.sep + if not rel_old.endswith(os.sep): + m[rel_old + os.sep] = rel_new + os.sep + return m diff --git a/TELF/pipeline/blocks/__init__.py b/TELF/pipeline/blocks/__init__.py index 8a71656b..69b73654 100644 --- a/TELF/pipeline/blocks/__init__.py +++ b/TELF/pipeline/blocks/__init__.py @@ -59,4 +59,7 @@ from .ocelot_filter_block import OcelotFilterBlock from .auto_bunny_simple_block import AutoBunnySimpleBlock from .term_table_block import TermTableBlock -from .spacey_NER_block import SpacyNERBlock \ No newline at end of file +from .spacey_NER_block import SpacyNERBlock +from .collect_hnmfk_leaf_block import CollectHNMFkLeafBlock +from .termite_neo4j_block import TermiteNeo4jBlock +from .termite_vector_block import TermiteVectorBlock diff --git a/TELF/pipeline/blocks/artic_fox_block.py b/TELF/pipeline/blocks/artic_fox_block.py index 160915f5..8e9fc59d 100644 --- a/TELF/pipeline/blocks/artic_fox_block.py +++ b/TELF/pipeline/blocks/artic_fox_block.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Dict, Sequence, Any, Tuple from .base_block import AnimalBlock -from .data_bundle import DataBundle +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY from ...post_processing import ArcticFox from ...factorization import HNMFk @@ -52,16 +52,41 @@ def __init__( **kw, ) + def run(self, bundle: DataBundle) -> None: - df = self.load_path(bundle[self.needs[0]]) + df = self.load_path(bundle[self.needs[0]]) vocabulary = self.load_path(bundle[self.needs[1]]) - model = HNMFk(experiment_name=bundle[self.needs[2]]) - model.load_model() # Loads model from the provided experiment_name path - pipeline = ArcticFox( - model=model, - **self.init_settings - ) - pipeline.run_full_pipeline(data_df = df, - vocab = vocabulary, - **self.call_settings) - bundle[f"{self.tag}.{self.provides[0]}"] = "Done" \ No newline at end of file + + raw_model_path = str(bundle[self.needs[2]]) + # Try to resolve to an absolute path for traceability; fall back to the raw string. + try: + resolved_model_path = str(Path(raw_model_path).expanduser().resolve()) + except Exception: + resolved_model_path = raw_model_path + + model = HNMFk(experiment_name=raw_model_path) + model.load_model() + + pipeline = ArcticFox(model=model, **self.init_settings) + # pipeline.run_full_pipeline(data_df=df, vocab=vocabulary, **self.call_settings) + + status_value = "Done" + + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + status_file = out_dir / "status.txt" + # Include model path info in the checkpointed status file + status_file.write_text( + f"status: {status_value}\n" + f"model_path: {raw_model_path}\n" + f"resolved_model_path: {resolved_model_path}\n", + encoding="utf-8", + ) + self.register_checkpoint(self.provides[0], status_file) + + bundle[f"{self.tag}.{self.provides[0]}"] = status_value + + + + \ No newline at end of file diff --git a/TELF/pipeline/blocks/base_block.py b/TELF/pipeline/blocks/base_block.py index 81d2c96e..5549a92e 100644 --- a/TELF/pipeline/blocks/base_block.py +++ b/TELF/pipeline/blocks/base_block.py @@ -7,6 +7,8 @@ from PIL import Image import pickle from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +import os, re, builtins +from contextlib import contextmanager Cond = Callable[[DataBundle, "AnimalBlock"], bool] @@ -52,6 +54,8 @@ def __init__( self._ckpt_keys = set(checkpoint_keys) if checkpoint_keys else set(self.provides) + + if hasattr(self.__class__, "CANONICAL_NEEDS"): self._canonical_needs = tuple(self.__class__.CANONICAL_NEEDS) else: @@ -114,6 +118,8 @@ def run(self, bundle: DataBundle) -> None: ... def __call__(self, bundle: DataBundle) -> DataBundle: + Path(bundle[SAVE_DIR_BUNDLE_KEY]).mkdir(parents=True, exist_ok=True) + # -------------------------------------------------------------- # 0) maybe load from checkpoint and bail early # -------------------------------------------------------------- @@ -124,7 +130,13 @@ def __call__(self, bundle: DataBundle) -> DataBundle: if exist: # hydrate only the keys we cached for k in self._ckpt_keys: - bundle[f"{self.tag}.{k}"] = self.load_path(ckpt[k]) + path = ckpt[k] + try: + bundle[f"{self.tag}.{k}"] = self.load_path(path) + except Exception: + # If reading (e.g., empty CSV) fails, just provide the path string. + # This still counts as "loaded", so run() is skipped. + bundle[f"{self.tag}.{k}"] = path # load any 'provides' from the class that is not saved to disk self._after_checkpoint_skip(bundle) @@ -150,7 +162,8 @@ def __call__(self, bundle: DataBundle) -> DataBundle: # 2) run block # -------------------------------------------------------------- self._pending_ckpt_map: Dict[str, str] = {} - self.run(bundle) + with self._io_path_rewriter(): + self.run(bundle) # -------------------------------------------------------------- # 3) verify outputs @@ -320,4 +333,111 @@ def copy(instance: Any, new_obj.needs = needs if provides: new_obj.provides = provides - return new_obj \ No newline at end of file + return new_obj + + + + def _base_tag(self, tag: str) -> str: + return re.sub(r'^\d+_', '', tag) + + @contextmanager + def _io_path_rewriter(self): + """ + While this block runs, rewrite any filesystem path segment matching + '/_/' (or '\\_\\') to the current self.tag, + for BOTH reads and writes. Also intercept directory creation to prevent + re-creating old numbered dirs. + """ + base_tag = self._base_tag(self.tag) + target_tag = self.tag + + # match start/middle/end occurrences for both / and \ + fwd_middle = re.compile(rf'(?<=/)\d+_{re.escape(base_tag)}(?=/)') + fwd_head = re.compile(rf'^\d+_{re.escape(base_tag)}(?=/)') + fwd_tail = re.compile(rf'(?<=/)\d+_{re.escape(base_tag)}$') + + bsl_middle = re.compile(rf'(?<=\\)\d+_{re.escape(base_tag)}(?=\\)') + bsl_head = re.compile(rf'^\d+_{re.escape(base_tag)}(?=\\)') + bsl_tail = re.compile(rf'(?<=\\)\d+_{re.escape(base_tag)}$') + + def _rewrite_seg(path_str: str) -> str: + s = path_str + s = fwd_middle.sub(target_tag, s) + s = fwd_head.sub(target_tag, s) + s = fwd_tail.sub(target_tag, s) + s = bsl_middle.sub(target_tag, s) + s = bsl_head.sub(target_tag, s) + s = bsl_tail.sub(target_tag, s) + return s + + # save originals + orig_open = builtins.open + orig_exists = os.path.exists + orig_isfile = os.path.isfile + orig_isdir = os.path.isdir + orig_mkdir = os.mkdir + orig_makedirs = os.makedirs + orig_path_mkdir = Path.mkdir + + def _open_patched(path, mode='r', *args, **kwargs): + try: + p = os.fspath(path) + except TypeError: + return orig_open(path, mode, *args, **kwargs) + p2 = _rewrite_seg(p) + # for reads: prefer rewritten, then fall back + if 'r' in mode and not any(ch in mode for ch in 'wax+'): + try: + return orig_open(p2, mode, *args, **kwargs) + except FileNotFoundError: + return orig_open(p, mode, *args, **kwargs) + # for writes or mixed modes: always use rewritten to avoid recreating old dirs + return orig_open(p2, mode, *args, **kwargs) + + def _wrap_exists(fn): + def _f(path): + try: + p = os.fspath(path) + except TypeError: + return fn(path) + return fn(_rewrite_seg(p)) + return _f + + def _mkdir_patched(path, mode=0o777): + p = _rewrite_seg(os.fspath(path)) + return orig_mkdir(p, mode) + + def _makedirs_patched(path, mode=0o777, exist_ok=False): + p = _rewrite_seg(os.fspath(path)) + return orig_makedirs(p, mode=mode, exist_ok=exist_ok) + + def _path_mkdir_patched(self_path: Path, mode=0o777, parents=False, exist_ok=False): + # emulate Path.mkdir but create at the rewritten location + p = _rewrite_seg(str(self_path)) + if parents: + return orig_makedirs(p, mode=mode, exist_ok=exist_ok) + else: + try: + return orig_mkdir(p, mode) + except FileExistsError: + if exist_ok: + return + raise + + try: + builtins.open = _open_patched + os.path.exists = _wrap_exists(orig_exists) + os.path.isfile = _wrap_exists(orig_isfile) + os.path.isdir = _wrap_exists(orig_isdir) + os.mkdir = _mkdir_patched + os.makedirs = _makedirs_patched + Path.mkdir = _path_mkdir_patched + yield + finally: + builtins.open = orig_open + os.path.exists = orig_exists + os.path.isfile = orig_isfile + os.path.isdir = orig_isdir + os.mkdir = orig_mkdir + os.makedirs = orig_makedirs + Path.mkdir = orig_path_mkdir diff --git a/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py b/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py new file mode 100644 index 00000000..9f33b617 --- /dev/null +++ b/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py @@ -0,0 +1,447 @@ +# TELF/pipeline/blocks/leaf_data_labels_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple, Optional, List +import os +import re +import pickle +import pandas as pd +import numpy as np +from datetime import datetime + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class CollectHNMFkLeafBlock(AnimalBlock): + """ + Build two artifacts from a completed HNMFk run: + + • LEAF_DATA.csv + ALL columns from the input df for documents in each LEAF node, + plus: cluster (int), Graph_Name (e.g. 'depth_2_Parent_1_0') + + • LEAF_LABELS.csv + Columns: Graph_Name, label, words + (words are comma-joined from top_words.csv for each cluster) + + • summary.txt + Total documents, number of leaf clusters, and per-cluster counts. + + Resolution order for HNMFk experiment directory: + 1) From bundle key (default: 'SemanticHNMFk.model_path', overridable) + 2) call_settings['hnmfk_dir'] + 3) Auto-discover under SAVE_DIR_BUNDLE_KEY using tag (default: 'SemanticHNMFk') + + Optional call_settings: + - hnmfk_dir: explicit path to the experiment directory + - hnmfk_tag: tag name to look for (default: 'SemanticHNMFk') + - hnmfk_bundle_key: bundle key for model path (default: f'{hnmfk_tag}.model_path') + + Provides: + - 'leaf_data_csv' → Path to LEAF_DATA.csv + - 'leaf_labels_csv' → Path to LEAF_LABELS.csv + (summary.txt is written alongside these outputs) + """ + + CANONICAL_NEEDS: Tuple[str, ...] = ("df", SAVE_DIR_BUNDLE_KEY) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("leaf_data_csv", "leaf_labels_csv"), + tag: str = "LeafDataLabels", + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + verbose: bool = True, + **kw: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=init_settings or {}, + call_settings=call_settings or {}, + verbose=verbose, + checkpoint=True, + load_checkpoint=False, # force a real run the first time + **kw, + ) + + # ───────────────────────── helpers ───────────────────────── + + @staticmethod + def _to_list(x, *, as_int=False, as_str=False) -> List[Any]: + if x is None: + out = [] + elif isinstance(x, list): + out = x + elif isinstance(x, (tuple, set)): + out = list(x) + elif hasattr(x, "tolist"): + out = x.tolist() + else: + try: + out = list(x) + except TypeError: + out = [x] + if as_int: + res = [] + for v in out: + try: + res.append(int(v)) + except Exception: + pass + return res + if as_str: + return [str(v) for v in out] + return out + + @staticmethod + def _find_semantic_dir(root: Path, tag_name: str = "SemanticHNMFk") -> Optional[Path]: + root = root.resolve() + candidates = [p for p in root.glob(f"*_{tag_name}") if p.is_dir()] + if candidates: + candidates.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return candidates[0].resolve() + plain = (root / tag_name) + return plain.resolve() if plain.is_dir() else None + + @staticmethod + def _load_pickle(path: Path): + with path.open("rb") as f: + return pickle.load(f) + + @staticmethod + def _load_checkpoint(exp_dir: Path) -> Dict[str, Any]: + """Try common checkpoint names; if not found, search recursively for 'checkpoint*'.""" + exp_dir = exp_dir.resolve() + # Primary expected name + ckpt = (exp_dir / "checkpoint.p").resolve() + candidates: List[Path] = [] + if ckpt.is_file(): + candidates = [ckpt] + else: + alternates = [ + exp_dir / "checkpoint.pkl", + exp_dir / "checkpoint.pickle", + exp_dir / "checkpoint", + ] + candidates = [c.resolve() for c in alternates if c.is_file()] + if not candidates: + globbed = [g.resolve() for g in exp_dir.glob("checkpoint*") if g.is_file()] + if globbed: + candidates = globbed + if not candidates: + # NEW: recursive search fallback + deep = [g.resolve() for g in exp_dir.rglob("checkpoint*") if g.is_file()] + if deep: + # newest first + deep.sort(key=lambda p: p.stat().st_mtime, reverse=True) + candidates = [deep[0]] + + if not candidates: + return {} + + ckpt = candidates[0] + try: + # FIX: correct class reference + return CollectHNMFkLeafBlock._load_pickle(ckpt) + except Exception: + return {} + + @staticmethod + def _safe_rebase(path: str, old_base: Optional[str], new_base: Optional[str]) -> str: + if not path: + return path + p_norm = os.path.normpath(path) + if not old_base or not new_base: + return str(Path(p_norm).resolve()) + try: + old_base_n = os.path.normpath(old_base) + new_base_n = os.path.normpath(new_base) + if p_norm.startswith(old_base_n): + rel = os.path.relpath(p_norm, old_base_n) + return str(Path(os.path.join(new_base_n, rel)).resolve()) + except Exception: + pass + try: + old_seg = re.search(r"(\d+_SemanticHNMFk|SemanticHNMFk)", str(old_base)).group(1) + new_seg = re.search(r"(\d+_SemanticHNMFk|SemanticHNMFk)", str(new_base)).group(1) + return str(Path(p_norm.replace(old_seg, new_seg)).resolve()) + except Exception: + return str(Path(p_norm).resolve()) + + def _log(self, fp: Path, *msgs: str) -> None: + try: + ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + fp.parent.mkdir(parents=True, exist_ok=True) + with fp.open("a", encoding="utf-8") as f: + for m in msgs: + f.write(f"[{ts}] {m}\n") + except Exception: + pass + + # ─────────────────────────── run ──────────────────────────── + + def run(self, bundle: DataBundle) -> None: + df: pd.DataFrame = bundle["df"] + root_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]).expanduser().resolve() + + out_dir = (root_dir / self.tag).resolve() + out_dir.mkdir(parents=True, exist_ok=True) + log_fp = out_dir / "debug.log" + + # Output paths (exact filenames requested) + leaf_data_csv = out_dir / "LEAF_DATA.csv" + leaf_labels_csv = out_dir / "LEAF_LABELS.csv" + summary_txt = out_dir / "summary.txt" # NEW + + # Resolve HNMFk experiment dir — ORDER: bundle → call_settings → auto + tag_name = self.call_settings.get("hnmfk_tag", "SemanticHNMFk") + bundle_key = self.call_settings.get("hnmfk_bundle_key", f"{tag_name}.model_path") + + exp_dir: Optional[Path] = None + src = "" + + # 1) from bundle (model_path preferred, else model object) + if bundle_key in bundle: + exp_dir = Path(bundle[bundle_key]).expanduser().resolve() + src = f"bundle[{bundle_key}]" + elif f"{tag_name}.model" in bundle: + try: + model_obj = bundle[f"{tag_name}.model"] + exp_dir = Path(getattr(model_obj, "experiment_save_path")).expanduser().resolve() + src = f"bundle[{tag_name}.model]" + except Exception: + exp_dir = None + + # 2) explicit in call_settings + if exp_dir is None and self.call_settings.get("hnmfk_dir"): + exp_dir = Path(self.call_settings["hnmfk_dir"]).expanduser().resolve() + src = "call_settings.hnmfk_dir" + + # 3) auto-discover next to SAVE_DIR + if exp_dir is None: + exp_dir = self._find_semantic_dir(root_dir, tag_name=tag_name) + src = f"auto({root_dir})" + + if not exp_dir or not exp_dir.exists(): + self._log(log_fp, f"HNMFk dir not found (src={src}). Writing empty outputs.") + pd.DataFrame(columns=list(df.columns) + ["cluster", "Graph_Name"]).to_csv(leaf_data_csv, index=False, encoding="utf-8-sig") + pd.DataFrame(columns=["Graph_Name", "label", "words"]).to_csv(leaf_labels_csv, index=False, encoding="utf-8-sig") + # NEW: still write an empty summary.txt + with summary_txt.open("w", encoding="utf-8") as f: + f.write("Total documents: 0\nLeaf clusters: 0\n") + self.register_checkpoint("leaf_data_csv", leaf_data_csv) + self.register_checkpoint("leaf_labels_csv", leaf_labels_csv) + bundle[f"{self.tag}.leaf_data_csv"] = leaf_data_csv + bundle[f"{self.tag}.leaf_labels_csv"] = leaf_labels_csv + if self.verbose: + print(f"[{self.tag}] Wrote:\n {leaf_data_csv}\n {leaf_labels_csv}\n {summary_txt}") + return + + ckpt = self._load_checkpoint(exp_dir) + if not ckpt: + self._log(log_fp, f"No checkpoint file in {exp_dir}. Writing empty outputs.") + pd.DataFrame(columns=list(df.columns) + ["cluster", "Graph_Name"]).to_csv(leaf_data_csv, index=False, encoding="utf-8-sig") + pd.DataFrame(columns=["Graph_Name", "label", "words"]).to_csv(leaf_labels_csv, index=False, encoding="utf-8-sig") + with summary_txt.open("w", encoding="utf-8") as f: + f.write("Total documents: 0\nLeaf clusters: 0\n") + self.register_checkpoint("leaf_data_csv", leaf_data_csv) + self.register_checkpoint("leaf_labels_csv", leaf_labels_csv) + bundle[f"{self.tag}.leaf_data_csv"] = leaf_data_csv + bundle[f"{self.tag}.leaf_labels_csv"] = leaf_labels_csv + if self.verbose: + print(f"[{self.tag}] Wrote:\n {leaf_data_csv}\n {leaf_labels_csv}\n {summary_txt}") + return + + node_save_paths: Dict[str, str] = ckpt.get("node_save_paths", {}) or {} + root_name: str = ckpt.get("root_name") or "Root" + old_base = ckpt.get("experiment_name") or ckpt.get("experiment_save_path") + new_base = str(exp_dir) + + final_df_parts: List[pd.DataFrame] = [] + label_parts: List[pd.DataFrame] = [] + visited = 0 + leaves_seen = 0 + + def _resolve(p: str) -> str: + return self._safe_rebase(p, old_base, new_base) + + def _load_node_safely(path: Path): + try: + return self._load_pickle(path) + except Exception as e: + self._log(log_fp, f"Failed to load node: {path} :: {e}") + return None + + def visit(node_name: str): + nonlocal visited, leaves_seen + if node_name not in node_save_paths: + self._log(log_fp, f"node_save_paths missing entry for '{node_name}'") + return + node_path = Path(_resolve(node_save_paths[node_name])).expanduser().resolve() + visited += 1 + if not node_path.exists(): + self._log(log_fp, f"node pickle missing: {node_path}") + return + + node_obj = _load_node_safely(node_path) + if node_obj is None: + return + + # children + for cname in self._to_list(getattr(node_obj, "child_node_names", []), as_str=True): + visit(cname) + + # leaf? + if not bool(getattr(node_obj, "leaf", False)): + return + + leaves_seen += 1 + original_indices = self._to_list(getattr(node_obj, "original_indices", None), as_int=True) + if not original_indices: + self._log(log_fp, f"leaf with empty original_indices: {node_path}") + return + + # extract ALL columns for those rows + try: + node_df = df.iloc[original_indices].copy() + except Exception as e: + self._log(log_fp, f"iloc failed for indices (len={len(original_indices)}): {e}") + return + + # num_clusters from W or signature + W = getattr(node_obj, "W", None) + if W is None: + sig = getattr(node_obj, "signature", None) + if sig is None: + num_clusters = 1 + else: + sig = np.asarray(sig) + if sig.ndim == 1: + sig = sig.reshape(-1, 1) + num_clusters = int(sig.shape[1]) + else: + W = np.asarray(W) + if W.ndim == 1: + W = W.reshape(-1, 1) + num_clusters = int(W.shape[1]) + + node_dir = node_path.parent + cluster_csv = node_dir / f"cluster_for_k={num_clusters}.csv" + cluster_membership = None + if cluster_csv.is_file(): + try: + cluster_membership = pd.read_csv(cluster_csv) + except Exception as e: + self._log(log_fp, f"read_csv failed: {cluster_csv} :: {e}") + + # attach cluster column + if isinstance(cluster_membership, pd.DataFrame) and "cluster" in cluster_membership.columns: + if len(cluster_membership) == len(node_df): + node_df["cluster"] = cluster_membership["cluster"].to_numpy() + else: + # try to align if cluster CSV has a 'doc_index' column + if "doc_index" in cluster_membership.columns: + try: + aligned = cluster_membership.set_index("doc_index").loc[original_indices] + node_df["cluster"] = aligned["cluster"].to_numpy() + except Exception: + m = min(len(cluster_membership), len(node_df)) + node_df = node_df.iloc[:m].copy() + node_df["cluster"] = cluster_membership["cluster"].iloc[:m].to_numpy() + self._log(log_fp, f"cluster length mismatch; truncated to {m} :: {cluster_csv}") + else: + m = min(len(cluster_membership), len(node_df)) + node_df = node_df.iloc[:m].copy() + node_df["cluster"] = cluster_membership["cluster"].iloc[:m].to_numpy() + self._log(log_fp, f"cluster length mismatch; truncated to {m} :: {cluster_csv}") + else: + node_df["cluster"] = 0 # default single cluster + + # Graph_Name from parent dir of node pickle + cluster id + graph_name_part = os.path.basename(os.path.dirname(str(node_path))) + for k_val in sorted(pd.unique(node_df["cluster"])): + sub = node_df[node_df["cluster"] == k_val].copy() + sub["Graph_Name"] = f"{graph_name_part}_{k_val}" + final_df_parts.append(sub) + + # labels + cs_fp = node_dir / "cluster_summaries.csv" + tw_fp = node_dir / "top_words.csv" + if cs_fp.is_file() and tw_fp.is_file(): + try: + cs = pd.read_csv(cs_fp) + tw = pd.read_csv(tw_fp) + words_map: Dict[str, str] = {} + for col in tw.columns: + words_map[str(col)] = ",".join(tw[col].dropna().astype(str).tolist()) + cs = cs.copy() + cs["cluster"] = cs["cluster"].astype(str) + cs["words"] = cs["cluster"].map(words_map).fillna("") + cs["Graph_Name"] = cs["cluster"].apply(lambda c: f"{graph_name_part}_{c}") + label_parts.append(cs[["Graph_Name", "label", "words"]]) + except Exception as e: + self._log(log_fp, f"label build failed at {node_dir}: {e}") + + # Walk from root + if root_name not in node_save_paths: + self._log(log_fp, f"root_name '{root_name}' missing in node_save_paths; nothing to traverse.") + else: + visit(root_name) + + # Concatenate & write (always write something) + if final_df_parts: + final_df = pd.concat(final_df_parts, ignore_index=True) + else: + final_df = pd.DataFrame(columns=list(df.columns) + ["cluster", "Graph_Name"]) + final_df.to_csv(leaf_data_csv, index=False, encoding="utf-8-sig") + + if label_parts: + labels_df = pd.concat(label_parts, ignore_index=True) + else: + labels_df = pd.DataFrame(columns=["Graph_Name", "label", "words"]) + labels_df.to_csv(leaf_labels_csv, index=False, encoding="utf-8-sig") + + # NEW: Write summary.txt (totals + per-cluster counts) + try: + if not final_df.empty and "Graph_Name" in final_df.columns: + counts = final_df.groupby("Graph_Name").size().sort_values(ascending=False) + else: + counts = pd.Series(dtype=int) + total_docs = int(len(final_df)) + num_leaf_clusters = int(len(counts)) + lines = [ + f"Total documents: {total_docs}", + f"Leaf clusters: {num_leaf_clusters}", + "", + ] + lines += [f"{name}\t{int(cnt)}" for name, cnt in counts.items()] + with summary_txt.open("w", encoding="utf-8") as f: + f.write("\n".join(lines)) + except Exception as e: + self._log(log_fp, f"Failed to write summary.txt: {e}") + + # Checkpoint + bundle exposure + self.register_checkpoint("leaf_data_csv", leaf_data_csv) + self.register_checkpoint("leaf_labels_csv", leaf_labels_csv) + bundle[f"{self.tag}.leaf_data_csv"] = leaf_data_csv + bundle[f"{self.tag}.leaf_labels_csv"] = leaf_labels_csv + # Optional: also expose summary path in the bundle (no need to register) + bundle[f"{self.tag}.summary_txt"] = summary_txt # NEW + + # Summary in logs + stdout (if verbose) + self._log( + log_fp, + f"exp_dir={exp_dir}", + f"nodes_seen={visited}, leaves_seen={leaves_seen}", + f"wrote LEAF_DATA rows={len(final_df)}", + f"wrote LEAF_LABELS rows={len(labels_df)}", + f"wrote summary at {summary_txt}", + ) + if self.verbose: + print(f"[{self.tag}] Wrote:\n {leaf_data_csv}\n {leaf_labels_csv}\n {summary_txt}") diff --git a/TELF/pipeline/blocks/peacock_stats_block.py b/TELF/pipeline/blocks/peacock_stats_block.py index 8a290e0f..db9591f8 100644 --- a/TELF/pipeline/blocks/peacock_stats_block.py +++ b/TELF/pipeline/blocks/peacock_stats_block.py @@ -67,7 +67,9 @@ def __init__( def run(self, bundle: DataBundle) -> None: # 1) Inputs & cleanup df: pd.DataFrame = bundle["df"] - out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) + # Save everything under the block's tag directory (like other blocks) + root_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) + out_dir = root_dir / self.tag out_dir.mkdir(parents=True, exist_ok=True) # ensure affiliation column is string @@ -230,9 +232,7 @@ def run(self, bundle: DataBundle) -> None: # 7) Dummy checkpoint under your single `provides` key if SAVE_DIR_BUNDLE_KEY in bundle: - ckpt_dir = out_dir / self.tag - ckpt_dir.mkdir(parents=True, exist_ok=True) - final_csv = ckpt_dir / "none.csv" + final_csv = out_dir / "none.csv" final_csv.write_text("") # empty placeholder self.register_checkpoint(self.provides[0], final_csv) bundle[f"{self.tag}.{self.provides[0]}"] = final_csv diff --git a/TELF/pipeline/blocks/termite_neo4j_block.py b/TELF/pipeline/blocks/termite_neo4j_block.py new file mode 100644 index 00000000..5f756d4f --- /dev/null +++ b/TELF/pipeline/blocks/termite_neo4j_block.py @@ -0,0 +1,417 @@ +# TELF/pipeline/blocks/termite_neo4j_block.py +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple, Optional +import pandas as pd +from copy import deepcopy +import ast +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +# --- Termite + constants (as in your notebook) --- +from TELF.applications import Termite +from TELF.applications.Termite.neo4j_termite import ( + ENTITY, RETURN_TYPE, ATTRIBUTES, ET, YEAR_TYPE, FROM_COL, DOCUMENT_TYPE, + ROW_INDEX, ATTR_COL, ATTR_NAME, TT, R, DOCUMENT_YEAR_RELATION, HT, + AUTHOR_DOCUMENT_RELATION, AUTHOR_ID_TYPE, EXTRACT_H, DOCUMENT_CITES_RELATION, + DOCUMENT_CITED_RELATION, DOCUMENT_TYPE_SCOPUS, EXTRACT_T, PAIRING, HEAD_TO_MANY, + TOPIC_TYPE, MAKE_ID_UNIQUE, KEYWORD_TYPE, INDEX_PAIRING, + DOCUMENT_PUBLISHER_RELATION, PUBLISHER, AFFILIATION_IDENTIFIER_TYPE, COUNTRY_TYPE, + CATEGORY, ACRONYM, RETREIVAL, ATTR_FUNC, ARGS, DOCUMENT_AFFILITATION_RELATION, + AFFILIATION_COUNTRY_RELATION, DOCUMENT_CATEGORY_RELATION, DOCUMENT_ACRONYM_RELATION +) + +# ---------- helpers (exactly your notebook’s logic) ---------- +def list_split_no_attrs(data, split_with=';'): + returnable_entities = [] + if isinstance(data, str): + for entity_value in data.split(split_with): + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = entity_value + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def get_cites(args): + data_string = args['data'] + return list_split_no_attrs(data_string.citations) + +def get_cited(args): + data_string = args['data'] + return list_split_no_attrs(data_string.references) + +def get_authors_ID(args): + data_string = args['data'] + data = data_string.s2_author_ids + if isinstance(data, str): + split_author_ids = data.split(';') + authors = data_string.s2_authors + if isinstance(authors, str): + authors_split_values = authors.split(';') + else: + authors_split_values = [] + returnable_entities = [] + for entity_value, attribute in zip(split_author_ids, authors_split_values): + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = entity_value + entity_returnable[ATTRIBUTES] = [('name', attribute)] + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + + +def get_parent_topic(args): + data_string = args['data'] + returnable_entities = [] + parent_name = data_string.parent_name + if type(parent_name) == str: + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = parent_name + entity_returnable['attributes'] = [("Graph_Name",parent_name)] + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def get_topic_keywords(args): + data_string = args['data'] + returnable_entities = [] + keyword_list = data_string.words + if isinstance(keyword_list, str): + keyword_list = ast.literal_eval(keyword_list) + for keyword in keyword_list: + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = keyword + returnable_entities.append(entity_returnable) + if returnable_entities: + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def get_affiliations(args): + returnable_entities = [] + affil_string = args['data'].affiliations + if type(affil_string) == str and affil_string != 'nan': + for k, v in ast.literal_eval(affil_string).items(): + if isinstance(v, dict): + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = k + name = v['name'] + entity_returnable[ATTRIBUTES] = [('name', name)] + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def get_countries(args): + data_string = args['data'] + returnable_entities = [] + affil_string = data_string.affiliations + if type(affil_string) == str and affil_string != 'nan': + for k, v in ast.literal_eval(data_string.affiliations).items(): + if isinstance(v, dict): + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = v['country'] + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def get_categories(args): + data_string = args['data'] + returnable_entities = [] + subject_areas = data_string.subject_areas + if type(subject_areas) == str: + split_subjects= subject_areas.split(';') + for subject in split_subjects: + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = subject + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def split_string(args, split_with= ';'): + data_string = args['data'] + return data_string.split(split_with) + +def list_split_no_attrs(data, split_with=';'): + returnable_entities = [] + if type(data) == str: + split_values = data.split(split_with) + for entity_value in split_values: + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = entity_value.strip() + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +def get_authors_ID(args): + data_string = args['data'] + data = data_string.author_ids + if type(data) == str: + split_author_ids= data.split(';') + authors = data_string.authors + if type(authors) == str: + authors_split_values = authors.split(';') + returnable_entities = [] + for entity_value, attribute in zip(split_author_ids, authors_split_values ): + entity_returnable = deepcopy(RETURN_TYPE) + entity_returnable[ENTITY] = entity_value + entity_returnable[ATTRIBUTES] = [('name', attribute)] + returnable_entities.append(entity_returnable) + return returnable_entities + else: + return [deepcopy(RETURN_TYPE)] + +# def get_acronyms(args): +# data_string = args['data'] +# return list_split_no_attrs(data_string.acronym_attribution, split_with=', ') +def get_acronyms(args): + """ + Extract acronym strings from a row, tolerating missing columns. + Tries columns in order: 'acronym_attribution', 'acronyms', 'acronym'. + Returns [] when nothing is present so no triples are created. + """ + row = args.get('data', None) + if row is None: + return [] + + candidates = ('acronym_attribution', 'acronyms', 'acronym') + + def _get_from_series(r, key): + try: + # pandas Series: prefer dict-style to avoid AttributeError when missing + if hasattr(r, 'get'): + return r.get(key, None) + # fallback for objects with attributes + return getattr(r, key, None) + except Exception: + return None + + value = None + for col in candidates: + v = _get_from_series(row, col) + if v is not None and str(v).strip() and str(v).lower() != 'nan': + value = v + break + + if not value: + return [] # no acronyms -> no triples + + # Accept either comma- or semicolon-separated values; normalize to commas first + text = str(value).replace(';', ',') + return list_split_no_attrs(text, split_with=',') + + + +def default_topic_triplet_map(): + topics_triplet_map_keywords = { + 'ENTITIES':[ + {ET:TOPIC_TYPE, MAKE_ID_UNIQUE:True, FROM_COL: 'Graph_Name', + ATTR_COL:[ + {FROM_COL: 'label', ATTR_NAME:'label', }, + {FROM_COL: 'Graph_Name', ATTR_NAME:'Graph_Name', } + ] + }, + {ET:KEYWORD_TYPE, MAKE_ID_UNIQUE:True}, + ], + 'RELATIONS':[ + {HT:TOPIC_TYPE, R:'child_of', TT:TOPIC_TYPE, EXTRACT_T: get_parent_topic, }, + {HT:TOPIC_TYPE, R:'mentions', TT:KEYWORD_TYPE, EXTRACT_T: get_topic_keywords, }, + ] + } + return topics_triplet_map_keywords + +def default_data_triplet_map(): + data_triplet_map_keywords = { + 'ENTITIES':[ + {ET:TOPIC_TYPE, MAKE_ID_UNIQUE:True, FROM_COL: 'Graph_Name'}, + {ET:DOCUMENT_TYPE, FROM_COL: "doi", + ATTR_COL:[ + {FROM_COL: 'title', ATTR_NAME:'Title', }, + {FROM_COL: 'eid', ATTR_NAME:'EID', }, + {FROM_COL: 's2id', ATTR_NAME:'S2ID', }, + {FROM_COL: 'doi', ATTR_NAME:'DOI', }, + ], + MAKE_ID_UNIQUE:True + }, + {ET:AFFILIATION_IDENTIFIER_TYPE, MAKE_ID_UNIQUE:True}, + {ET:COUNTRY_TYPE, MAKE_ID_UNIQUE:True}, + # {ET:DOCUMENT_TYPE_SCOPUS, MAKE_ID_UNIQUE:True}, + {ET:CATEGORY, MAKE_ID_UNIQUE:True}, + {ET:ACRONYM, MAKE_ID_UNIQUE:True}, + {ET:YEAR_TYPE, FROM_COL: 'year', ATTR_COL:None, ATTR_FUNC:None, MAKE_ID_UNIQUE:True}, + {ET:AUTHOR_ID_TYPE, FROM_COL: 'author_ids', + ATTR_COL:[{FROM_COL: 'authors', ATTR_NAME:'Author_Name', RETREIVAL: split_string, ARGS: None}], + ATTR_FUNC:split_string, ARGS: None, MAKE_ID_UNIQUE:True + }, + {ET:PUBLISHER, FROM_COL: 'publication_name', MAKE_ID_UNIQUE:True}, + ], + 'RELATIONS':[ + {HT:DOCUMENT_TYPE, R:'part_of_topic', TT:TOPIC_TYPE}, + {HT:DOCUMENT_TYPE, R:DOCUMENT_YEAR_RELATION, TT:YEAR_TYPE}, + # {HT:DOCUMENT_TYPE, R:DOCUMENT_TOPIC_RELATION, TT:TOPIC_TYPE, EXTRACT_T: get_absolute_cluster,}, + {HT:AUTHOR_ID_TYPE, R:AUTHOR_DOCUMENT_RELATION, TT:DOCUMENT_TYPE, EXTRACT_H: get_authors_ID}, + {HT:DOCUMENT_TYPE, R:DOCUMENT_AFFILITATION_RELATION, TT:AFFILIATION_IDENTIFIER_TYPE, EXTRACT_T: get_affiliations}, + {HT:AFFILIATION_IDENTIFIER_TYPE, R:AFFILIATION_COUNTRY_RELATION, TT:COUNTRY_TYPE, EXTRACT_H: get_affiliations, EXTRACT_T: get_countries, PAIRING: INDEX_PAIRING}, + # {HT:DOCUMENT_TYPE, R:DOCUMENT_CITES_RELATION, TT:DOCUMENT_TYPE_SCOPUS, EXTRACT_T: get_cites, PAIRING: HEAD_TO_MANY}, + # {HT:DOCUMENT_TYPE, R:DOCUMENT_CITED_RELATION, TT:DOCUMENT_TYPE_SCOPUS, EXTRACT_T: get_cited, PAIRING: HEAD_TO_MANY}, + {HT:DOCUMENT_TYPE, R:DOCUMENT_PUBLISHER_RELATION, TT:PUBLISHER}, + {HT:DOCUMENT_TYPE, R:DOCUMENT_CATEGORY_RELATION, TT:CATEGORY, EXTRACT_T: get_categories, PAIRING: HEAD_TO_MANY}, + {HT:DOCUMENT_TYPE, R:DOCUMENT_ACRONYM_RELATION, TT:ACRONYM, EXTRACT_T: get_acronyms}, + ] + } + return data_triplet_map_keywords + +# ---------- block with defaults ---------- +DEFAULT_CALL_SETTINGS: Dict[str, Any] = { + # Source CSVs (None → try bundle keys) + "raw_csv_path": None, # data/docs + "topic_csv_path": None, # topics/labels (NEW) + + # Output files + "triplets_filename": "triplets.csv", # legacy: data triplets filename + "data_triplets_filename": None, # NEW (falls back to triplets_filename) + "topic_triplets_filename": "topic_triplets.csv", # NEW + + # Column → triplet mappings + "column_triplet_map": None, # legacy: data map + "column_triplet_map_data": None, # NEW (falls back to column_triplet_map or default_data_triplet_map()) + "column_triplet_map_topics": None, # NEW (falls back to default_topic_triplet_map()) + + # Neo4j creds: env → fallback to local dev defaults + "neo4j_uri": os.getenv("NEO4J_URI", "neo4j://localhost:7666"), + "neo4j_user": os.getenv("NEO4J_USER", "neo4j"), + "neo4j_pass": os.getenv("NEO4J_PASS", "local_password"), + # Optional auth/token passthrough for Termite + "token": None, +} + +class TermiteNeo4jBlock(AnimalBlock): + """ + Wrapper that: + 1) builds *data* triplets from a CSV and pushes to Neo4j + 2) builds *topic* triplets from a CSV and pushes to Neo4j + """ + + CANONICAL_NEEDS: Tuple[str, ...] = ("leaf_data_csv", "leaf_labels_csv") + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("data_triplets_csv", "topic_triplets_csv", ), + tag: str = "TermiteNeo4j", + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + verbose: bool = True, + **kw: Any, + ) -> None: + # Merge provided call_settings over defaults + merged_call_settings = {**DEFAULT_CALL_SETTINGS, **(call_settings or {})} + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=init_settings or {}, + call_settings=merged_call_settings, + verbose=verbose, + checkpoint=False, + **kw, + ) + + def _prefer_bundle(self, bundle: DataBundle, *keys: str) -> Optional[str]: + """Try bundle.get across multiple keys; return first truthy value or None.""" + for k in keys: + try: + v = bundle.get(k) + except Exception: + v = None + if v: + return v + return None + + def run(self, bundle: DataBundle) -> None: + root_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]).expanduser().resolve() + out_dir = (root_dir / self.tag).resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + # ---------- Resolve CSV inputs ---------- + data_csv_path = ( + self.call_settings.get("raw_csv_path") + or self._prefer_bundle(bundle, "LeafDataLabels.leaf_data_csv", "leaf_data_csv") + ) + if not data_csv_path: + raise RuntimeError("[TermiteNeo4j] 'raw_csv_path' not provided and no leaf_data_csv found in bundle.") + data_csv_path = Path(str(data_csv_path)).expanduser().resolve() + + topic_csv_path = ( + self.call_settings.get("topic_csv_path") + or self._prefer_bundle(bundle, "LeafDataLabels.leaf_labels_csv", "leaf_labels_csv") + or data_csv_path # final fallback if topics are embedded in same CSV + ) + topic_csv_path = Path(str(topic_csv_path)).expanduser().resolve() + + # ---------- Resolve outputs ---------- + data_triplets_filename = ( + self.call_settings.get("data_triplets_filename") + or self.call_settings.get("triplets_filename", "triplets.csv") + ) + topic_triplets_filename = self.call_settings.get("topic_triplets_filename", "topic_triplets.csv") + + data_triplets_path = out_dir / data_triplets_filename + topic_triplets_path = out_dir / topic_triplets_filename + + # ---------- Resolve triplet maps ---------- + data_triplet_map = ( + self.call_settings.get("column_triplet_map_data") + or self.call_settings.get("column_triplet_map") + or default_data_triplet_map() + ) + topic_triplet_map = self.call_settings.get("column_triplet_map_topics") or default_topic_triplet_map() + + # ---------- Neo4j / Termite ---------- + neo4j_uri = self.call_settings["neo4j_uri"] + neo4j_user = self.call_settings["neo4j_user"] + neo4j_pass = self.call_settings["neo4j_pass"] + token = self.call_settings.get("token", None) + + termite = Termite( + kg_credentials=(neo4j_uri, (neo4j_user, neo4j_pass)), + vector_uri=None, + db_nme="default", + token=token, + verbose=self.verbose, + ) + + # Create uniqueness constraints for both schemas up front + termite.make_unique_constrains(data_triplet_map) + termite.make_unique_constrains(topic_triplet_map) + + # ---------- PASS 1: DATA triplets ---------- + termite.from_csv_to_triplets(str(data_csv_path), str(data_triplets_path), data_triplet_map) + termite.update_database_multithreaded(str(data_triplets_path)) + + # ---------- PASS 2: TOPIC triplets ---------- + termite.from_csv_to_triplets(str(topic_csv_path), str(topic_triplets_path), topic_triplet_map) + termite.update_database_multithreaded(str(topic_triplets_path)) + + # ---------- Register outputs ---------- + # Back-compat: keep ".triplets_csv" pointing to the DATA triplets + self.register_checkpoint("data_triplets_csv", data_triplets_path) + self.register_checkpoint("topic_triplets_csv", topic_triplets_path) + # self.register_checkpoint("triplets_csv", data_triplets_path) + + bundle[f"{self.tag}.data_triplets_csv"] = data_triplets_path + bundle[f"{self.tag}.topic_triplets_csv"] = topic_triplets_path + # bundle[f"{self.tag}.triplets_csv"] = data_triplets_path # alias + + if self.verbose: + print(f"[{self.tag}] Data triplets @ {data_triplets_path}") + print(f"[{self.tag}] Topic triplets @ {topic_triplets_path}") diff --git a/TELF/pipeline/blocks/termite_vector_block.py b/TELF/pipeline/blocks/termite_vector_block.py new file mode 100644 index 00000000..f4612a19 --- /dev/null +++ b/TELF/pipeline/blocks/termite_vector_block.py @@ -0,0 +1,137 @@ +# TELF/pipeline/blocks/termite_vector_index_block.py +from __future__ import annotations +import os +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple, Optional, List + +import pandas as pd +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from TELF.applications import Termite + +class TermiteVectorBlock(AnimalBlock): + """ + Mirrors test_termite_e2e.py: + os.environ[...] for OpenSearch + t = Termite(kg_credentials=None, verbose=True, model_name=MODEL) + emb_map = t.compute_embeddings(df, model_name=MODEL) + embeddings = [emb_map[i] for i in df.index] + t.store.ensure_index(index, dim, metric='cosine') + t.store.upsert(index, ids, embeddings, payloads=[{'text': ...}, ...]) + (optional) test search with a query string + + call_settings: + raw_csv_path: str|Path # default: bundle['LeafDataLabels.leaf_data_csv'] + id_column: str # default: 'eid' (falls back to df.index if absent) + text_column: str # default: 'abstract' + index_name: str # default: 'termite_vectors' + model_name: str # default: 'malteos/scincl' + metric: str # default: 'cosine' + env: dict # optional overrides for OS_* and EMBEDDING_STORE + test_query_text: str # optional; if set, do a top-k search + test_k: int # default: 5 + + Provides: + - '{tag}.vector_index_name' + - '{tag}.vector_stats' (docs, dim, metric, index) + - '{tag}.search_hits' (if test_query_text provided) + """ + + CANONICAL_NEEDS: Tuple[str, ...] = ("leaf_data_csv",) + + def __init__(self, *, needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("vector_index_name", "vector_stats"), + tag: str = "TermiteVectorIndex", + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + verbose: bool = True, **kw: Any) -> None: + super().__init__(needs=needs, provides=provides, tag=tag, + init_settings=init_settings or {}, call_settings=call_settings or {}, + verbose=verbose, checkpoint=False, **kw) + + # ---- env like your script ---- + def _init_env(self): + env = { + "EMBEDDING_STORE": "opensearch", + "OS_HOST": os.getenv("OS_HOST", "localhost"), + "OS_PORT": os.getenv("OS_PORT", "9200"), + "OS_USE_SSL": os.getenv("OS_USE_SSL", "false"), + } + overrides = self.call_settings.get("env") or {} + env.update(overrides) + for k, v in env.items(): + os.environ[str(k)] = str(v) + + def _load_df(self, bundle: DataBundle) -> pd.DataFrame: + raw = self.call_settings.get("raw_csv_path") or bundle.get("LeafDataLabels.leaf_data_csv") + if raw: + return pd.read_csv(Path(raw).expanduser().resolve()) + # fallback to bundle df if caller wired it that way + return bundle["df"] + + def run(self, bundle: DataBundle) -> None: + self._init_env() + df = self._load_df(bundle) + + model = self.call_settings.get("model_name", "malteos/scincl") + t = Termite(kg_credentials=None, verbose=self.verbose, model_name=model) + + # embeddings aligned to df.index + emb_map = t.compute_embeddings(df, model_name=model) + embeddings = [emb_map[i] for i in df.index] + if not embeddings: + raise RuntimeError(f"[{self.tag}] No embeddings produced") + dim = len(embeddings[0]) + + index_name = self.call_settings.get("index_name", "termite_vectors") + metric = self.call_settings.get("metric", "cosine") + + # ensure index dimension & metric + t.store.ensure_index(index=index_name, dim=dim, metric=metric) + + # ids + payloads + id_col = self.call_settings.get("id_column", "eid") + ids: List[str] + if id_col in df.columns: + ids = df[id_col].astype(str).tolist() + else: + ids = df.index.astype(str).tolist() # fallback + + text_col = self.call_settings.get("text_column", "abstract") + if text_col not in df.columns: + raise RuntimeError(f"[{self.tag}] text_column '{text_col}' not in DataFrame") + payloads = [{"text": txt} for txt in df[text_col].astype(str).tolist()] + + # upsert + t.store.upsert(index_name, ids, embeddings, payloads=payloads) + + # optional quick search + hits_out = None + qtxt = self.call_settings.get("test_query_text") + if qtxt: + # same embed path as your helper + _qdf = pd.DataFrame({text_col: [qtxt]}) + qmap = t.compute_embeddings(_qdf, model_name=model) + qvec = qmap[_qdf.index[0]] + k = int(self.call_settings.get("test_k", 5)) + hits = t.store.search(index_name, qvec, k=k, source_fields="id,text") + # normalize output for bundle + hits_out = [{"id": _id, "score": float(score), "text": (src or {}).get("text")} for _id, score, src in hits] + bundle[f"{self.tag}.search_hits"] = hits_out + if self.verbose: + print(f"[{self.tag}] Top-{k} search preview:") + for h in hits_out: + print(f" {h['score']:.4f} id={h['id']} text={h['text']}") + + # expose in bundle + bundle[f"{self.tag}.vector_index_name"] = index_name + bundle[f"{self.tag}.vector_stats"] = { + "docs": int(len(df)), + "dim": int(dim), + "metric": metric, + "index": index_name, + "id_column": id_col, + "text_column": text_col, + } + if self.verbose: + print(f"[{self.tag}] Upserted {len(df)} vectors (dim={dim}) into '{index_name}'") diff --git a/TELF/pipeline/blocks/wolf_block.py b/TELF/pipeline/blocks/wolf_block.py index 46af8ee5..f30535c5 100644 --- a/TELF/pipeline/blocks/wolf_block.py +++ b/TELF/pipeline/blocks/wolf_block.py @@ -96,6 +96,9 @@ def __init__( def run(self, bundle: DataBundle) -> None: # ─── 1) Load the DataFrame & map ────────────────────────────────────── df = self.load_path(bundle[self.needs[0]]) + print("Number of rows in df:", len(df)) + print("Unique IDs:", df[self.category_map[self.category]['col']].nunique()) + orca_map = bundle[self.needs[1]] OUTPUT_DIR = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag diff --git a/examples/Full TELF Pipeline/single_block_examples/post_process_example.ipynb b/examples/Full TELF Pipeline/single_block_examples/post_process_example.ipynb index c1eac4b6..10982289 100644 --- a/examples/Full TELF Pipeline/single_block_examples/post_process_example.ipynb +++ b/examples/Full TELF Pipeline/single_block_examples/post_process_example.ipynb @@ -11,7 +11,16 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/barron/anaconda3/envs/TELF/lib/python3.11/site-packages/pymilvus/client/__init__.py:6: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", + " from pkg_resources import DistributionNotFound, get_distribution\n" + ] + } + ], "source": [ "from TELF.pipeline import BlockManager\n", "from TELF.pipeline.blocks import (\n", @@ -253,33 +262,75 @@ "output_type": "stream", "text": [ "▶ [1/14] VultureClean …\n", - "✓ [1/14] VultureClean finished in 0.01s\n", + "✓ [1/14] VultureClean finished in 2.16s\n", "▶ [2/14] BeaverVocab …\n", "✓ [2/14] BeaverVocab finished in 0.01s\n", "▶ [3/14] BeaverDW …\n", "✓ [3/14] BeaverDW finished in 0.01s\n", "▶ [4/14] NMFk …\n", - "✓ [4/14] NMFk finished in 3.00s\n", + "✓ [4/14] NMFk finished in 2.51s\n", "▶ [5/14] NMFAnalyzer …\n", - "✓ [5/14] NMFAnalyzer finished in 21.28s\n", - "▶ [6/14] NMFLabels …\n", - "✓ [6/14] NMFLabels finished in 13.13s\n", - "▶ [7/14] HNMFk …\n", - "✓ [7/14] HNMFk finished in 0.00s\n", - "▶ [8/14] HNMFAnalyzer …\n", - "✓ [8/14] HNMFAnalyzer finished in 0.00s\n", - "▶ [9/14] HNMFkLabels …\n", - "✓ [9/14] HNMFkLabels finished in 17.50s\n", - "▶ [10/14] NoClusterAnalyzer …\n", - "✓ [10/14] NoClusterAnalyzer finished in 1.21s\n", - "▶ [11/14] NoClusterLabels …\n", - "✓ [11/14] NoClusterLabels finished in 0.97s\n", - "▶ [12/14] LoadDF …\n", - "✓ [12/14] LoadDF finished in 0.00s\n", - "▶ [13/14] ClusterOnlyAnalyzer …\n", - "✓ [13/14] ClusterOnlyAnalyzer finished in 13.45s\n", - "▶ [14/14] ClusterOnlyLabels …\n", - "✓ [14/14] ClusterOnlyLabels finished in 6.35s\n" + "✓ [5/14] NMFAnalyzer finished in 29.23s\n", + "▶ [6/14] NMFLabels …\n" + ] + }, + { + "ename": "SSLError", + "evalue": "(MaxRetryError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /malteos/scincl/resolve/main/tokenizer_config.json (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)')))\"), '(Request ID: 686934cb-bb27-4d01-b270-95dcc54a900d)')", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mSSLCertVerificationError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connectionpool.py:464\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 463\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m464\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_validate_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 465\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connectionpool.py:1093\u001b[39m, in \u001b[36mHTTPSConnectionPool._validate_conn\u001b[39m\u001b[34m(self, conn)\u001b[39m\n\u001b[32m 1092\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m conn.is_closed:\n\u001b[32m-> \u001b[39m\u001b[32m1093\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1095\u001b[39m \u001b[38;5;66;03m# TODO revise this, see https://github.com/urllib3/urllib3/issues/2791\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connection.py:790\u001b[39m, in \u001b[36mHTTPSConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 788\u001b[39m server_hostname_rm_dot = server_hostname.rstrip(\u001b[33m\"\u001b[39m\u001b[33m.\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m--> \u001b[39m\u001b[32m790\u001b[39m sock_and_verified = \u001b[43m_ssl_wrap_socket_and_match_hostname\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[43m=\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mcert_reqs\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcert_reqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mssl_version\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mssl_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mssl_minimum_version\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mssl_minimum_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mssl_maximum_version\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mssl_maximum_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mca_certs\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mca_certs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43mcert_file\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcert_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m \u001b[49m\u001b[43mkey_file\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mkey_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 801\u001b[39m \u001b[43m \u001b[49m\u001b[43mkey_password\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mkey_password\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserver_hostname_rm_dot\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 803\u001b[39m \u001b[43m \u001b[49m\u001b[43mssl_context\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mssl_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 804\u001b[39m \u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 805\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_hostname\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43massert_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 806\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_fingerprint\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43massert_fingerprint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 807\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 808\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = sock_and_verified.socket\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connection.py:969\u001b[39m, in \u001b[36m_ssl_wrap_socket_and_match_hostname\u001b[39m\u001b[34m(sock, cert_reqs, ssl_version, ssl_minimum_version, ssl_maximum_version, cert_file, key_file, key_password, ca_certs, ca_cert_dir, ca_cert_data, assert_hostname, assert_fingerprint, server_hostname, ssl_context, tls_in_tls)\u001b[39m\n\u001b[32m 967\u001b[39m server_hostname = normalized\n\u001b[32m--> \u001b[39m\u001b[32m969\u001b[39m ssl_sock = \u001b[43mssl_wrap_socket\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 970\u001b[39m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[43m=\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 971\u001b[39m \u001b[43m \u001b[49m\u001b[43mkeyfile\u001b[49m\u001b[43m=\u001b[49m\u001b[43mkey_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 972\u001b[39m \u001b[43m \u001b[49m\u001b[43mcertfile\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcert_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 973\u001b[39m \u001b[43m \u001b[49m\u001b[43mkey_password\u001b[49m\u001b[43m=\u001b[49m\u001b[43mkey_password\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 974\u001b[39m \u001b[43m \u001b[49m\u001b[43mca_certs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mca_certs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 975\u001b[39m \u001b[43m \u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 976\u001b[39m \u001b[43m \u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[43m=\u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 977\u001b[39m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 978\u001b[39m \u001b[43m \u001b[49m\u001b[43mssl_context\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 979\u001b[39m \u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 980\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 982\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/util/ssl_.py:480\u001b[39m, in \u001b[36mssl_wrap_socket\u001b[39m\u001b[34m(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)\u001b[39m\n\u001b[32m 478\u001b[39m context.set_alpn_protocols(ALPN_PROTOCOLS)\n\u001b[32m--> \u001b[39m\u001b[32m480\u001b[39m ssl_sock = \u001b[43m_ssl_wrap_socket_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 481\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m ssl_sock\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/util/ssl_.py:524\u001b[39m, in \u001b[36m_ssl_wrap_socket_impl\u001b[39m\u001b[34m(sock, ssl_context, tls_in_tls, server_hostname)\u001b[39m\n\u001b[32m 522\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m SSLTransport(sock, ssl_context, server_hostname)\n\u001b[32m--> \u001b[39m\u001b[32m524\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mssl_context\u001b[49m\u001b[43m.\u001b[49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/ssl.py:517\u001b[39m, in \u001b[36mSSLContext.wrap_socket\u001b[39m\u001b[34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[39m\n\u001b[32m 511\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side=\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[32m 512\u001b[39m do_handshake_on_connect=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 513\u001b[39m suppress_ragged_eofs=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 514\u001b[39m server_hostname=\u001b[38;5;28;01mNone\u001b[39;00m, session=\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m 515\u001b[39m \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[32m 516\u001b[39m \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m517\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msslsocket_class\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 518\u001b[39m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[43m=\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 519\u001b[39m \u001b[43m \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 520\u001b[39m \u001b[43m \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 521\u001b[39m \u001b[43m \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m=\u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 522\u001b[39m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 523\u001b[39m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 524\u001b[39m \u001b[43m \u001b[49m\u001b[43msession\u001b[49m\u001b[43m=\u001b[49m\u001b[43msession\u001b[49m\n\u001b[32m 525\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/ssl.py:1104\u001b[39m, in \u001b[36mSSLSocket._create\u001b[39m\u001b[34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[39m\n\u001b[32m 1103\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m-> \u001b[39m\u001b[32m1104\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1105\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/ssl.py:1382\u001b[39m, in \u001b[36mSSLSocket.do_handshake\u001b[39m\u001b[34m(self, block)\u001b[39m\n\u001b[32m 1381\u001b[39m \u001b[38;5;28mself\u001b[39m.settimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m-> \u001b[39m\u001b[32m1382\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sslobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1383\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n", + "\u001b[31mSSLCertVerificationError\u001b[39m: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mSSLError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connectionpool.py:488\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 487\u001b[39m new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n\u001b[32m--> \u001b[39m\u001b[32m488\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m new_e\n\u001b[32m 490\u001b[39m \u001b[38;5;66;03m# conn.request() calls http.client.*.request, not the method in\u001b[39;00m\n\u001b[32m 491\u001b[39m \u001b[38;5;66;03m# urllib3.request. It also calls makefile (recv) on the socket.\u001b[39;00m\n", + "\u001b[31mSSLError\u001b[39m: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/requests/adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n", + "\u001b[31mMaxRetryError\u001b[39m: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /malteos/scincl/resolve/main/tokenizer_config.json (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)')))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mSSLError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[5]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m bundle = \u001b[43mmanager\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/block_manager.py:87\u001b[39m, in \u001b[36mBlockManager.__call__\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 85\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m contextlib.redirect_stdout(buf_out), contextlib.redirect_stderr(buf_err):\n\u001b[32m---> \u001b[39m\u001b[32m87\u001b[39m \u001b[38;5;28mself\u001b[39m.bundle = \u001b[43mblock\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mbundle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 88\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[32m 89\u001b[39m buf_err.write(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m⚠️ Exception in block \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mblock.tag\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/blocks/base_block.py:153\u001b[39m, in \u001b[36mAnimalBlock.__call__\u001b[39m\u001b[34m(self, bundle)\u001b[39m\n\u001b[32m 149\u001b[39m \u001b[38;5;66;03m# --------------------------------------------------------------\u001b[39;00m\n\u001b[32m 150\u001b[39m \u001b[38;5;66;03m# 2) run block\u001b[39;00m\n\u001b[32m 151\u001b[39m \u001b[38;5;66;03m# --------------------------------------------------------------\u001b[39;00m\n\u001b[32m 152\u001b[39m \u001b[38;5;28mself\u001b[39m._pending_ckpt_map: Dict[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = {}\n\u001b[32m--> \u001b[39m\u001b[32m153\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbundle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 155\u001b[39m \u001b[38;5;66;03m# --------------------------------------------------------------\u001b[39;00m\n\u001b[32m 156\u001b[39m \u001b[38;5;66;03m# 3) verify outputs\u001b[39;00m\n\u001b[32m 157\u001b[39m \u001b[38;5;66;03m# --------------------------------------------------------------\u001b[39;00m\n\u001b[32m 158\u001b[39m view = bundle.namespaced(\u001b[38;5;28mself\u001b[39m.tag)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/blocks/post_process_label_analysis_block.py:147\u001b[39m, in \u001b[36mLabelAnalyzerBlock.run\u001b[39m\u001b[34m(self, bundle)\u001b[39m\n\u001b[32m 145\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m csv_paths:\n\u001b[32m 146\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m csv_p \u001b[38;5;129;01min\u001b[39;00m csv_paths:\n\u001b[32m--> \u001b[39m\u001b[32m147\u001b[39m labels = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_label_single_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcsv_p\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabeler\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 148\u001b[39m result_dict[\u001b[38;5;28mstr\u001b[39m(csv_p)] = labels\n\u001b[32m 149\u001b[39m written_csvs.append(\u001b[38;5;28mstr\u001b[39m(csv_p.with_name(\u001b[33m\"\u001b[39m\u001b[33mlabels.csv\u001b[39m\u001b[33m\"\u001b[39m)))\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/blocks/post_process_label_analysis_block.py:96\u001b[39m, in \u001b[36mLabelAnalyzerBlock._label_single_csv\u001b[39m\u001b[34m(self, csv_path, labeler)\u001b[39m\n\u001b[32m 91\u001b[39m \u001b[38;5;66;03m# choose cluster strategy automatically\u001b[39;00m\n\u001b[32m 92\u001b[39m strat = (\u001b[33m\"\u001b[39m\u001b[33mcolumn\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.call_settings[\u001b[33m\"\u001b[39m\u001b[33mcluster_col\u001b[39m\u001b[33m\"\u001b[39m] \u001b[38;5;129;01min\u001b[39;00m df.columns\n\u001b[32m 93\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33msingle\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.call_settings[\u001b[33m\"\u001b[39m\u001b[33mcluster_strategy\u001b[39m\u001b[33m\"\u001b[39m] \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \\\n\u001b[32m 94\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m.call_settings[\u001b[33m\"\u001b[39m\u001b[33mcluster_strategy\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m---> \u001b[39m\u001b[32m96\u001b[39m labels = \u001b[43mlabeler\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlabel_texts\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 97\u001b[39m \u001b[43m \u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 98\u001b[39m \u001b[43m \u001b[49m\u001b[43mprovider\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mprovider\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 99\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmodel_name\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 100\u001b[39m \u001b[43m \u001b[49m\u001b[43mopenai_api_key\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mopenai_api_key\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 101\u001b[39m \u001b[43m \u001b[49m\u001b[43mcluster_strategy\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mstrat\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 102\u001b[39m \u001b[43m \u001b[49m\u001b[43mcluster_col\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcluster_col\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 103\u001b[39m \u001b[43m \u001b[49m\u001b[43mtop_n_words\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtop_n_words\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 104\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_candidates\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mnum_candidates\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 105\u001b[39m \u001b[43m \u001b[49m\u001b[43muse_gpu\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcall_settings\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43muse_gpu\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 106\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 108\u001b[39m \u001b[38;5;66;03m# write labels.csv next to input\u001b[39;00m\n\u001b[32m 109\u001b[39m out_csv = csv_path.with_name(\u001b[33m\"\u001b[39m\u001b[33mlabels.csv\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/post_processing/ArcticFox/label_analyzer.py:280\u001b[39m, in \u001b[36mLabelAnalyzer.label_texts\u001b[39m\u001b[34m(self, data, provider, model_name, openai_api_key, cluster_strategy, cluster_col, top_n_words, num_candidates, use_gpu)\u001b[39m\n\u001b[32m 276\u001b[39m crit = \u001b[38;5;28mself\u001b[39m._crit()\n\u001b[32m 277\u001b[39m model = model_name \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[32m 278\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mllama3.2:3b-instruct-fp16\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m provider==\u001b[33m\"\u001b[39m\u001b[33mollama\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mgpt-3.5-turbo\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 279\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m280\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mlabel_clusters\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 281\u001b[39m \u001b[43m \u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkw_df\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 282\u001b[39m \u001b[43m \u001b[49m\u001b[43mprovider\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprovider\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 283\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 284\u001b[39m \u001b[43m \u001b[49m\u001b[43mapi_key\u001b[49m\u001b[43m=\u001b[49m\u001b[43mopenai_api_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 285\u001b[39m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_candidates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 286\u001b[39m \u001b[43m \u001b[49m\u001b[43mcrit\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcrit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 287\u001b[39m \u001b[43m \u001b[49m\u001b[43mgpu\u001b[49m\u001b[43m=\u001b[49m\u001b[43muse_gpu\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 288\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/post_processing/ArcticFox/label_analyzer.py:224\u001b[39m, in \u001b[36mLabelAnalyzer.label_clusters\u001b[39m\u001b[34m(self, df, kw_df, provider, model, api_key, k, crit, gpu)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mlabel_clusters\u001b[39m(\n\u001b[32m 213\u001b[39m \u001b[38;5;28mself\u001b[39m,\n\u001b[32m 214\u001b[39m df: pd.DataFrame,\n\u001b[32m (...)\u001b[39m\u001b[32m 222\u001b[39m gpu: \u001b[38;5;28mbool\u001b[39m,\n\u001b[32m 223\u001b[39m ) -> Dict[\u001b[38;5;28mint\u001b[39m,\u001b[38;5;28mstr\u001b[39m]:\n\u001b[32m--> \u001b[39m\u001b[32m224\u001b[39m centres = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_centres\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgpu\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 225\u001b[39m dist_kw = \u001b[38;5;28mself\u001b[39m._distinctive(df)\n\u001b[32m 226\u001b[39m out: Dict[\u001b[38;5;28mint\u001b[39m,\u001b[38;5;28mstr\u001b[39m] = {}\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/post_processing/ArcticFox/label_analyzer.py:162\u001b[39m, in \u001b[36mLabelAnalyzer._centres\u001b[39m\u001b[34m(self, df, gpu)\u001b[39m\n\u001b[32m 161\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_centres\u001b[39m(\u001b[38;5;28mself\u001b[39m, df: pd.DataFrame, gpu: \u001b[38;5;28mbool\u001b[39m) -> Dict[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mtuple\u001b[39m]:\n\u001b[32m--> \u001b[39m\u001b[32m162\u001b[39m emb = \u001b[43mcompute_embeddings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 163\u001b[39m cents = compute_centroids(emb, df) \u001b[38;5;66;03m# ← pass df here\u001b[39;00m\n\u001b[32m 164\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m {\n\u001b[32m 165\u001b[39m cid: closest_embedding_to_centroid(emb, c, metric=\u001b[38;5;28mself\u001b[39m.distance_metric)\n\u001b[32m 166\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m cid, c \u001b[38;5;129;01min\u001b[39;00m cents.items() \u001b[38;5;28;01mif\u001b[39;00m c \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 167\u001b[39m }\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/helpers/embeddings.py:28\u001b[39m, in \u001b[36mcompute_embeddings\u001b[39m\u001b[34m(df, model_name, cols, sep_token, as_np, use_gpu)\u001b[39m\n\u001b[32m 27\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompute_embeddings\u001b[39m(df, *, model_name=\u001b[33m'\u001b[39m\u001b[33mSCINCL\u001b[39m\u001b[33m'\u001b[39m, cols=[\u001b[33m'\u001b[39m\u001b[33mtitle\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mabstract\u001b[39m\u001b[33m'\u001b[39m], sep_token=\u001b[33m'\u001b[39m\u001b[33m[SEP]\u001b[39m\u001b[33m'\u001b[39m, as_np=\u001b[38;5;28;01mFalse\u001b[39;00m, use_gpu=\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[32m---> \u001b[39m\u001b[32m28\u001b[39m tokenizer, model, device = \u001b[43mget_transformer_llm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muse_gpu\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 29\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m use_gpu \u001b[38;5;129;01mand\u001b[39;00m device == \u001b[33m'\u001b[39m\u001b[33mcpu\u001b[39m\u001b[33m'\u001b[39m:\n\u001b[32m 30\u001b[39m warnings.warn(\u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mTried to use GPU, but GPU is not available. Using \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdevice\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.\u001b[39m\u001b[33m'\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/helpers/llm_models.py:59\u001b[39m, in \u001b[36mget_transformer_llm\u001b[39m\u001b[34m(embedding_model, device)\u001b[39m\n\u001b[32m 57\u001b[39m device = \u001b[33m'\u001b[39m\u001b[33mcuda:0\u001b[39m\u001b[33m'\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m device \u001b[38;5;129;01mand\u001b[39;00m torch.cuda.is_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[33m'\u001b[39m\u001b[33mcpu\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 58\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mUsing device: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdevice\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m tokenizer = \u001b[43mAutoTokenizer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[43mget_model_name\u001b[49m\u001b[43m(\u001b[49m\u001b[43membedding_model\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 60\u001b[39m model = AutoModel.from_pretrained(get_model_name(embedding_model)).to(device)\n\u001b[32m 61\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m tokenizer, model, device\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/transformers/models/auto/tokenization_auto.py:881\u001b[39m, in \u001b[36mAutoTokenizer.from_pretrained\u001b[39m\u001b[34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[39m\n\u001b[32m 878\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)\n\u001b[32m 880\u001b[39m \u001b[38;5;66;03m# Next, let's try to use the tokenizer_config file to get the tokenizer class.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m881\u001b[39m tokenizer_config = \u001b[43mget_tokenizer_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m tokenizer_config:\n\u001b[32m 883\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m] = tokenizer_config[\u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m]\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/transformers/models/auto/tokenization_auto.py:713\u001b[39m, in \u001b[36mget_tokenizer_config\u001b[39m\u001b[34m(pretrained_model_name_or_path, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, **kwargs)\u001b[39m\n\u001b[32m 710\u001b[39m token = use_auth_token\n\u001b[32m 712\u001b[39m commit_hash = kwargs.get(\u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m--> \u001b[39m\u001b[32m713\u001b[39m resolved_config_file = \u001b[43mcached_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 714\u001b[39m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 715\u001b[39m \u001b[43m \u001b[49m\u001b[43mTOKENIZER_CONFIG_FILE\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 716\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 717\u001b[39m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 718\u001b[39m \u001b[43m \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 719\u001b[39m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 720\u001b[39m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 721\u001b[39m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 722\u001b[39m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 723\u001b[39m \u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m=\u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 724\u001b[39m \u001b[43m \u001b[49m\u001b[43m_raise_exceptions_for_gated_repo\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 725\u001b[39m \u001b[43m \u001b[49m\u001b[43m_raise_exceptions_for_missing_entries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 726\u001b[39m \u001b[43m \u001b[49m\u001b[43m_raise_exceptions_for_connection_errors\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 727\u001b[39m \u001b[43m \u001b[49m\u001b[43m_commit_hash\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcommit_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 728\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 729\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m resolved_config_file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 730\u001b[39m logger.info(\u001b[33m\"\u001b[39m\u001b[33mCould not locate the tokenizer configuration file, will try to use the model config instead.\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/transformers/utils/hub.py:342\u001b[39m, in \u001b[36mcached_file\u001b[39m\u001b[34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[39m\n\u001b[32m 339\u001b[39m user_agent = http_user_agent(user_agent)\n\u001b[32m 340\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 341\u001b[39m \u001b[38;5;66;03m# Load from URL or cache if already cached\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m342\u001b[39m resolved_file = \u001b[43mhf_hub_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 343\u001b[39m \u001b[43m \u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 344\u001b[39m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 345\u001b[39m \u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[43m==\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m0\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 346\u001b[39m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 347\u001b[39m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 348\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 349\u001b[39m \u001b[43m \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[43m=\u001b[49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 350\u001b[39m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 351\u001b[39m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 352\u001b[39m \u001b[43m \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 353\u001b[39m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 354\u001b[39m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 355\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 356\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m GatedRepoError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 357\u001b[39m resolved_file = _get_cache_file_to_return(path_or_repo_id, full_filename, cache_dir, revision)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py:114\u001b[39m, in \u001b[36mvalidate_hf_hub_args.._inner_fn\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 111\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[32m 112\u001b[39m kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.\u001b[34m__name__\u001b[39m, has_token=has_token, kwargs=kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m114\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/file_download.py:1010\u001b[39m, in \u001b[36mhf_hub_download\u001b[39m\u001b[34m(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, user_agent, force_download, proxies, etag_timeout, token, local_files_only, headers, endpoint, resume_download, force_filename, local_dir_use_symlinks)\u001b[39m\n\u001b[32m 990\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _hf_hub_download_to_local_dir(\n\u001b[32m 991\u001b[39m \u001b[38;5;66;03m# Destination\u001b[39;00m\n\u001b[32m 992\u001b[39m local_dir=local_dir,\n\u001b[32m (...)\u001b[39m\u001b[32m 1007\u001b[39m local_files_only=local_files_only,\n\u001b[32m 1008\u001b[39m )\n\u001b[32m 1009\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1010\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_hf_hub_download_to_cache_dir\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1011\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Destination\u001b[39;49;00m\n\u001b[32m 1012\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1013\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# File info\u001b[39;49;00m\n\u001b[32m 1014\u001b[39m \u001b[43m \u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1015\u001b[39m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1016\u001b[39m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1017\u001b[39m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1018\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# HTTP info\u001b[39;49;00m\n\u001b[32m 1019\u001b[39m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m=\u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1020\u001b[39m \u001b[43m \u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1021\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_headers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1022\u001b[39m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1023\u001b[39m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1024\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Additional options\u001b[39;49;00m\n\u001b[32m 1025\u001b[39m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1026\u001b[39m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1027\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/file_download.py:1073\u001b[39m, in \u001b[36m_hf_hub_download_to_cache_dir\u001b[39m\u001b[34m(cache_dir, repo_id, filename, repo_type, revision, endpoint, etag_timeout, headers, proxies, token, local_files_only, force_download)\u001b[39m\n\u001b[32m 1069\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m pointer_path\n\u001b[32m 1071\u001b[39m \u001b[38;5;66;03m# Try to get metadata (etag, commit_hash, url, size) from the server.\u001b[39;00m\n\u001b[32m 1072\u001b[39m \u001b[38;5;66;03m# If we can't, a HEAD request error is returned.\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1073\u001b[39m (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = \u001b[43m_get_metadata_or_catch_error\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1074\u001b[39m \u001b[43m \u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1075\u001b[39m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1076\u001b[39m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1077\u001b[39m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1078\u001b[39m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m=\u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1079\u001b[39m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1080\u001b[39m \u001b[43m \u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1081\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1082\u001b[39m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1083\u001b[39m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1084\u001b[39m \u001b[43m \u001b[49m\u001b[43mstorage_folder\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstorage_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1085\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelative_filename\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelative_filename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1086\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1088\u001b[39m \u001b[38;5;66;03m# etag can be None for several reasons:\u001b[39;00m\n\u001b[32m 1089\u001b[39m \u001b[38;5;66;03m# 1. we passed local_files_only.\u001b[39;00m\n\u001b[32m 1090\u001b[39m \u001b[38;5;66;03m# 2. we don't have a connection\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 1096\u001b[39m \u001b[38;5;66;03m# If the specified revision is a commit hash, look inside \"snapshots\".\u001b[39;00m\n\u001b[32m 1097\u001b[39m \u001b[38;5;66;03m# If the specified revision is a branch or tag, look inside \"refs\".\u001b[39;00m\n\u001b[32m 1098\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m head_call_error \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1099\u001b[39m \u001b[38;5;66;03m# Couldn't make a HEAD call => let's try to find a local file\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/file_download.py:1546\u001b[39m, in \u001b[36m_get_metadata_or_catch_error\u001b[39m\u001b[34m(repo_id, filename, repo_type, revision, endpoint, proxies, etag_timeout, headers, token, local_files_only, relative_filename, storage_folder)\u001b[39m\n\u001b[32m 1544\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 1545\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1546\u001b[39m metadata = \u001b[43mget_hf_file_metadata\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1547\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m=\u001b[49m\u001b[43mendpoint\u001b[49m\n\u001b[32m 1548\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1549\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m EntryNotFoundError \u001b[38;5;28;01mas\u001b[39;00m http_error:\n\u001b[32m 1550\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m storage_folder \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m relative_filename \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1551\u001b[39m \u001b[38;5;66;03m# Cache the non-existence of the file\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py:114\u001b[39m, in \u001b[36mvalidate_hf_hub_args.._inner_fn\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 111\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[32m 112\u001b[39m kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.\u001b[34m__name__\u001b[39m, has_token=has_token, kwargs=kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m114\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/file_download.py:1463\u001b[39m, in \u001b[36mget_hf_file_metadata\u001b[39m\u001b[34m(url, token, proxies, timeout, library_name, library_version, user_agent, headers, endpoint)\u001b[39m\n\u001b[32m 1460\u001b[39m hf_headers[\u001b[33m\"\u001b[39m\u001b[33mAccept-Encoding\u001b[39m\u001b[33m\"\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33midentity\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;66;03m# prevent any compression => we want to know the real size of the file\u001b[39;00m\n\u001b[32m 1462\u001b[39m \u001b[38;5;66;03m# Retrieve metadata\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1463\u001b[39m r = \u001b[43m_request_wrapper\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1464\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mHEAD\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 1465\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1466\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_headers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1467\u001b[39m \u001b[43m \u001b[49m\u001b[43mallow_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 1468\u001b[39m \u001b[43m \u001b[49m\u001b[43mfollow_relative_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 1469\u001b[39m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1470\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1471\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1472\u001b[39m hf_raise_for_status(r)\n\u001b[32m 1474\u001b[39m \u001b[38;5;66;03m# Return\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/file_download.py:286\u001b[39m, in \u001b[36m_request_wrapper\u001b[39m\u001b[34m(method, url, follow_relative_redirects, **params)\u001b[39m\n\u001b[32m 284\u001b[39m \u001b[38;5;66;03m# Recursively follow relative redirects\u001b[39;00m\n\u001b[32m 285\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m follow_relative_redirects:\n\u001b[32m--> \u001b[39m\u001b[32m286\u001b[39m response = \u001b[43m_request_wrapper\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 287\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 288\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 289\u001b[39m \u001b[43m \u001b[49m\u001b[43mfollow_relative_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 290\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 291\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 293\u001b[39m \u001b[38;5;66;03m# If redirection, we redirect only relative paths.\u001b[39;00m\n\u001b[32m 294\u001b[39m \u001b[38;5;66;03m# This is useful in case of a renamed repository.\u001b[39;00m\n\u001b[32m 295\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[32m300\u001b[39m <= response.status_code <= \u001b[32m399\u001b[39m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/file_download.py:309\u001b[39m, in \u001b[36m_request_wrapper\u001b[39m\u001b[34m(method, url, follow_relative_redirects, **params)\u001b[39m\n\u001b[32m 306\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m response\n\u001b[32m 308\u001b[39m \u001b[38;5;66;03m# Perform request and return if status_code is not in the retry list.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m309\u001b[39m response = \u001b[43mhttp_backoff\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretry_on_exceptions\u001b[49m\u001b[43m=\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretry_on_status_codes\u001b[49m\u001b[43m=\u001b[49m\u001b[43m(\u001b[49m\u001b[32;43m429\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 310\u001b[39m hf_raise_for_status(response)\n\u001b[32m 311\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/utils/_http.py:310\u001b[39m, in \u001b[36mhttp_backoff\u001b[39m\u001b[34m(method, url, max_retries, base_wait_time, max_wait_time, retry_on_exceptions, retry_on_status_codes, **kwargs)\u001b[39m\n\u001b[32m 307\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33mdata\u001b[39m\u001b[33m\"\u001b[39m].seek(io_obj_initial_pos)\n\u001b[32m 309\u001b[39m \u001b[38;5;66;03m# Perform request and return if status_code is not in the retry list.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m310\u001b[39m response = \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 311\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m retry_on_status_codes:\n\u001b[32m 312\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/huggingface_hub/utils/_http.py:96\u001b[39m, in \u001b[36mUniqueRequestIdAdapter.send\u001b[39m\u001b[34m(self, request, *args, **kwargs)\u001b[39m\n\u001b[32m 94\u001b[39m logger.debug(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mSend: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m_curlify(request)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m 95\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m96\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 97\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m requests.RequestException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 98\u001b[39m request_id = request.headers.get(X_AMZN_TRACE_ID)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/requests/adapters.py:675\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 671\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m ProxyError(e, request=request)\n\u001b[32m 673\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 674\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m675\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m 677\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 679\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[31mSSLError\u001b[39m: (MaxRetryError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /malteos/scincl/resolve/main/tokenizer_config.json (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)')))\"), '(Request ID: 686934cb-bb27-4d01-b270-95dcc54a900d)')" ] } ], @@ -289,117 +340,44 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "'BeaverDW': ['X']\n", - "'BeaverVocab': ['vocabulary']\n", - "'ClusterOnlyAnalyzer': ['clusters_path']\n", - "'ClusterOnlyLabels': ['result', 'label_paths']\n", - "'DataBundle': ['result_path']\n", - "'Default': ['df']\n", - "'HNMFAnalyzer': ['clusters_path']\n", - "'HNMFk': ['hnmfk_model', 'saved_path']\n", - "'HNMFkLabels': ['result', 'label_paths']\n", - "'Init': ['save_path', 'dir']\n", - "'LoadDF': ['df', 'df_paths']\n", - "'NMFAnalyzer': ['clusters_path']\n", - "'NMFLabels': ['result', 'label_paths']\n", - "'NMFk': ['nmfk_model', 'nmfk_model_path']\n", - "'NoClusterAnalyzer': ['clusters_path']\n", - "'NoClusterLabels': ['result', 'label_paths']\n", - "'VultureClean': ['df', 'vulture_steps']\n" - ] - } - ], + "outputs": [], "source": [ "bundle.print_tags_and_keys()" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "NamespaceView(tag='NMFLabels', keys=['result', 'label_paths'])" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bundle.NMFLabels\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'example_results/post_process_example/NMFk/cluster_for_k=20.csv': {17: 'Machine Learning for 19.0 Malware Detection Models',\n", - " 19: 'Quantum Inspired Neural Network Optimization Techniques',\n", - " 16: 'Reinforcement Learning for Robust Model Training',\n", - " 18: 'Anomaly Detection Using 18.0 Matrix Models',\n", - " 11: 'Malware Family Classification Using HNMFk Classifier Approach',\n", - " 6: 'Neural Architecture Search for Dense Matrix Optimization on GPU Clusters',\n", - " 9: 'Federated Learning for Collaborative Filtering Systems',\n", - " 13: 'Anomaly Detection',\n", - " 12: 'Malware Novelty Detection Using Hierarchical Tensor Factorization',\n", - " 8: 'Machine Learning'}}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bundle.NMFLabels.result\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "NamespaceView(tag='HNMFkLabels', keys=['result', 'label_paths'])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bundle.HNMFkLabels\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "dev_artic_fox", + "display_name": "TELF", "language": "python", "name": "python3" }, diff --git a/examples/Full TELF Pipeline/single_block_examples/semantic_hnmfk_collection_slurm_option.ipynb b/examples/Full TELF Pipeline/single_block_examples/semantic_hnmfk_collection_slurm_option.ipynb new file mode 100644 index 00000000..18bf8f50 --- /dev/null +++ b/examples/Full TELF Pipeline/single_block_examples/semantic_hnmfk_collection_slurm_option.ipynb @@ -0,0 +1,1730 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2ede8a29", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Slurm is not available.\n" + ] + } + ], + "source": [ + "import os\n", + "nnn = 1\n", + "os.environ[\"OMP_NUM_THREADS\"] = str(nnn) # export OMP_NUM_THREADS=1\n", + "os.environ[\"OPENBLAS_NUM_THREADS\"] = str(nnn) # export OPENBLAS_NUM_THREADS=1\n", + "os.environ[\"MKL_NUM_THREADS\"] = str(nnn) # export MKL_NUM_THREADS=1\n", + "os.environ[\"VECLIB_MAXIMUM_THREADS\"] = str(nnn) # export VECLIB_MAXIMUM_THREADS=1\n", + "os.environ[\"NUMEXPR_NUM_THREADS\"] = str(nnn) # export NUMEXPR_NUM_THREADS=1\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "import pandas as pd\n", + "from pathlib import Path\n", + "\n", + "import shutil\n", + "USE_SLURM = False\n", + "if shutil.which(\"squeue\"):\n", + " print(\"Slurm is available on this system.\")\n", + " USE_SLURM = True\n", + "else:\n", + " print(\"Slurm is not available.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "080bc0aa", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/barron/anaconda3/envs/TELF/lib/python3.11/site-packages/pymilvus/client/__init__.py:6: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", + " from pkg_resources import DistributionNotFound, get_distribution\n" + ] + } + ], + "source": [ + "from TELF.pipeline.blocks import DataBundle, SAVE_DIR_BUNDLE_KEY, SOURCE_DIR_BUNDLE_KEY\n", + "from TELF.pipeline import BlockManager \n", + "\n", + "from TELF.pipeline.blocks import (\n", + " DataBundle,\n", + " VultureCleanBlock,\n", + " BeaverVocabBlock,\n", + " OrcaBlock,\n", + " WolfBlock,\n", + " CleanDuplicatesBlock,\n", + " CleanAffiliationsBlock,\n", + " BeaverDocWordBlock,\n", + " SemanticHNMFkBlock,\n", + " ArticFoxBlock,\n", + " TermAttributionBlock,\n", + " LoadTermsBlock,\n", + " TermAttributionBlock,\n", + " SBatchBlock,\n", + " ClusteringAnalyzerBlock,\n", + " PeacockStatsBlock,\n", + " PipelineSummaryBlock,\n", + " CollectHNMFkLeafBlock,\n", + " TermiteNeo4jBlock,\n", + " TermiteVectorBlock,\n", + " TermTableBlock,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "065a0b92", + "metadata": {}, + "source": [ + "# Load Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ad35c75b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 50 entries, 0 to 49\n", + "Data columns (total 19 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 eid 50 non-null object \n", + " 1 s2id 50 non-null object \n", + " 2 doi 50 non-null object \n", + " 3 title 50 non-null object \n", + " 4 abstract 50 non-null object \n", + " 5 year 50 non-null int64 \n", + " 6 authors 50 non-null object \n", + " 7 author_ids 50 non-null object \n", + " 8 affiliations 50 non-null object \n", + " 9 funding 5 non-null object \n", + " 10 PACs 8 non-null object \n", + " 11 publication_name 50 non-null object \n", + " 12 subject_areas 50 non-null object \n", + " 13 s2_authors 50 non-null object \n", + " 14 s2_author_ids 50 non-null object \n", + " 15 citations 45 non-null object \n", + " 16 references 38 non-null object \n", + " 17 num_citations 50 non-null int64 \n", + " 18 num_references 50 non-null float64\n", + "dtypes: float64(1), int64(2), object(16)\n", + "memory usage: 7.6+ KB\n" + ] + } + ], + "source": [ + "df = pd.read_csv(Path(\"..\") / \"..\" / \"..\" /\"data\" / \"sample2.csv\").head(50)\n", + "EXAMPLE_OUTPUT = Path( \"example_results\") / 'semantic_HNMFk_collection_slurm_option' \n", + "bundle = DataBundle({'Default.df':df, \n", + " SAVE_DIR_BUNDLE_KEY: EXAMPLE_OUTPUT,\n", + " SOURCE_DIR_BUNDLE_KEY: EXAMPLE_OUTPUT})\n", + "df.info()\n" + ] + }, + { + "cell_type": "markdown", + "id": "7d417e2b", + "metadata": {}, + "source": [ + "# Build the Blocks" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[CleanDuplicates] needs → (df) provides → (df)\n", + "[Orca] needs → (df) provides → (df, map)\n", + "[CleanAffiliations] needs → (df) provides → (df)\n", + "[VultureClean] needs → (df, substitutions) provides → (df, vulture_steps)\n", + "[BeaverVocab] needs → (df) provides → (vocabulary)\n" + ] + } + ], + "source": [ + "duplicate_cleaner_block = CleanDuplicatesBlock()\n", + "orca_block = OrcaBlock()\n", + "clean_affiliations_block = CleanAffiliationsBlock()\n", + "vulture_block = VultureCleanBlock(verbose=True, \n", + " use_substitutions=True,\n", + " init_settings={\"n_jobs\":-1, 'parallel_backend': 'threading'})\n", + "vocab_block = BeaverVocabBlock(call_settings={'min_df':3, 'max_df':0.6, 'max_features':10000})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "57270163", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Terms] needs → (dir) provides → (terms, substitutions, substitutions_reverse, query)\n" + ] + }, + { + "data": { + "text/plain": [ + "{'decision-making': 'decision-making',\n", + " 'self-supervised learning': 'self-supervised_learning',\n", + " 'neural architecture search': 'neural_architecture_search',\n", + " 'edge computing': 'edge_computing',\n", + " 'mystery': 'mystery',\n", + " 'machine learning': 'machine_learning',\n", + " 'malware': 'malware',\n", + " 'ransomware': 'ransomware',\n", + " 'matrix': 'matrix',\n", + " 'anomaly': 'anomaly',\n", + " 'anomaly detection': 'anomaly_detection',\n", + " 'cluster analysis': 'cluster_analysis',\n", + " 'cluster': 'cluster',\n", + " 'unsupervised': 'unsupervised'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "terms_block = LoadTermsBlock( call_settings={SOURCE_DIR_BUNDLE_KEY: Path('../../../data/sample_terms3.md')})\n", + "out_terms = terms_block(bundle=bundle)\n", + "out_terms.substitutions\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "70063e68", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[DocWord] needs → (df, vocabulary) provides → (X)\n" + ] + } + ], + "source": [ + "matrix_block = BeaverDocWordBlock(tag=\"DocWord\", needs=(\"df\", \"vocabulary\",))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "43ea212c", + "metadata": {}, + "outputs": [], + "source": [ + "nmfk_params = {\n", + " \"n_perturbs\": 2,\n", + " \"n_iters\": 2,\n", + " \"epsilon\": 0.015,\n", + " \"n_jobs\": -1,\n", + " \"init\": \"nnsvd\",\n", + " \"use_gpu\": False,\n", + " \"save_output\": True,\n", + " \"collect_output\": True,\n", + " \"predict_k_method\": \"sill\",\n", + " \"verbose\": True,\n", + " \"nmf_verbose\": False,\n", + " \"transpose\": False,\n", + " \"sill_thresh\": 0.8,\n", + " \"pruned\": True,\n", + " \"nmf_method\": \"nmf_fro_mu\",\n", + " \"calculate_error\": True,\n", + " \"predict_k\": True,\n", + " \"use_consensus_stopping\": 0,\n", + " \"calculate_pac\": True,\n", + " \"consensus_mat\": True,\n", + " \"perturb_type\": \"uniform\",\n", + " \"perturb_multiprocessing\": False,\n", + " \"perturb_verbose\": False,\n", + " \"simple_plot\": True,\n", + " \"k_search_method\": \"bst_pre\",\n", + " \"H_sill_thresh\": 0.1,\n", + " \"clustering_method\": \"kmeans\",\n", + " \"device\": -1,\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SemanticHNMFk] needs → (DocWord.X, df, vocabulary) provides → (model, model_path)\n" + ] + } + ], + "source": [ + "semantic_hfactor_block = SemanticHNMFkBlock(\n", + " needs=(\"DocWord.X\", \"df\", \"vocabulary\", ),\n", + " init_settings={\n", + " \"depth\":2, \n", + " \"sample_thresh\":5,\n", + " \"Ks_deep_max\":30,\n", + " \"nmfk_params\":[nmfk_params],\n", + " },\n", + " call_settings={\n", + " \"Ks\":range(2, 10),\n", + " }\n", + ")\n", + "\n", + "if USE_SLURM:\n", + " sbatch_hnmfk = SBatchBlock(\n", + " wrapped_block=semantic_hfactor_block,\n", + " venv_type=\"conda\",\n", + " venv_path=\"TELF2\",\n", + " )\n", + "\n", + " hnmfk_block = sbatch_hnmfk\n", + "else:\n", + " hnmfk_block = semantic_hfactor_block\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[WolfAuthor] needs → (df, map) provides → (graph_co-author)\n", + "[WolfAffil] needs → (df, map) provides → (graph_co-affiliation)\n", + "[Attribution] needs → (df, terms) provides → (df, term_representation_df)\n", + "[ArticFox] needs → (df, vocabulary, model_path) provides → (block_status)\n" + ] + } + ], + "source": [ + "wolf_coauthor_block = WolfBlock(tag=\"WolfAuthor\", category='co-author')\n", + "wolf_coaffiliation_block = WolfBlock(tag=\"WolfAffil\", category='co-affiliation')\n", + "term_attribute_block = TermAttributionBlock( )\n", + "post_process_block = ArticFoxBlock(call_settings={\"ollama_model\":\"llama3.2:3b-instruct-fp16\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ab3a95f3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[HNMFAnalyzer] needs → (df, hnmfk_model, vocabulary) provides → (clusters_path)\n", + "[PipelineSummary] needs → (∅) provides → (docs_summary_df, docs_summary_plot)\n", + "[PeacockStats] needs → (df, save_path) provides → (outpath)\n", + "[LeafDataLabels] needs → (df, save_path) provides → (leaf_data_csv, leaf_labels_csv)\n" + ] + } + ], + "source": [ + "hnmfk_analyzer = ClusteringAnalyzerBlock(\n", + " tag='HNMFAnalyzer',\n", + " mode='hnmf'\n", + ")\n", + "\n", + "summary_block = PipelineSummaryBlock()\n", + "peacock_block = PeacockStatsBlock()\n", + "\n", + "collect_leaves = CollectHNMFkLeafBlock( call_settings={\"hnmfk_dir\": \"./example_results/semantic_HNMFk_collection_slurm_option/07_SemanticHNMFk\"},)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1f312726", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[TermiteNeo4j] needs → (leaf_data_csv, leaf_labels_csv) provides → (data_triplets_csv, topic_triplets_csv)\n", + "[TermiteVectorIndex] needs → (leaf_data_csv) provides → (vector_index_name, vector_stats)\n" + ] + } + ], + "source": [ + "neo4j_block = TermiteNeo4jBlock(call_settings={\n", + " \"raw_csv_path\": bundle.get(\"LeafDataLabels.leaf_data_csv\"),\n", + " \"neo4j_uri\": \"neo4j://localhost:7666\",\n", + " \"neo4j_user\": \"neo4j\",\n", + " \"neo4j_pass\": \"local_password\",\n", + "})\n", + "vector_store_block = TermiteVectorBlock(call_settings={\n", + " \"raw_csv_path\": bundle.get(\"LeafDataLabels.leaf_data_csv\"),\n", + " \"id_column\": \"eid\",\n", + " \"text_column\": \"abstract\",\n", + " \"index_name\": \"termite_vectors_test_e2e\",\n", + " \"model_name\": \"malteos/scincl\",\n", + " \"env\": { # optional override; defaults match your script\n", + " \"EMBEDDING_STORE\": \"opensearch\",\n", + " \"OS_HOST\": \"localhost\",\n", + " \"OS_PORT\": \"9200\",\n", + " \"OS_USE_SSL\": \"false\",\n", + " },\n", + " # Optional smoke test:\n", + " # \"test_query_text\": \"What problem in real-world malware labeling does the HNMFk Classifier aim to solve?\",\n", + " # \"test_k\": 5,\n", + "})" + ] + }, + { + "cell_type": "markdown", + "id": "94129c86", + "metadata": {}, + "source": [ + "# Block Manager" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94129c86", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "VultureClean – all needs met
Orca – all needs met
CleanAffiliations – all needs met
BeaverVocab – all needs met
DocWord – all needs met
SemanticHNMFk – all needs met
Attribution – all needs met
WolfAuthor – all needs met
WolfAffil – all needs met
ArticFox – all needs met
PeacockStats – all needs met
LeafDataLabels – all needs met
PipelineSummary – all needs met
TermiteNeo4j – all needs met
TermiteVectorIndex – all needs met" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Block (tag) │ Needs (✓/✗) │ Provides\n", + "──────────────────────────────────────────────────────────────────────────────────────\n", + "VultureCleanBlock (VultureClean) │ df, substitutions │ ['df', 'vulture_steps']\n", + "OrcaBlock (Orca) │ df │ ['df', 'map']\n", + "CleanAffiliationsBlock (CleanAffiliations) │ df │ ['df']\n", + "BeaverVocabBlock (BeaverVocab) │ df │ ['vocabulary']\n", + "BeaverDocWordBlock (DocWord) │ df, vocabulary │ ['X']\n", + "SemanticHNMFkBlock (SemanticHNMFk) │ DocWord.X, df, vocabulary │ ['model', 'model_path']\n", + "TermAttributionBlock (Attribution) │ df, terms │ ['df', 'term_representation_df']\n", + "WolfBlock (WolfAuthor) │ df, map │ ['graph_co-author']\n", + "WolfBlock (WolfAffil) │ df, map │ ['graph_co-affiliation']\n", + "ArticFoxBlock (ArticFox) │ df, vocabulary, model_path │ ['block_status']\n", + "PeacockStatsBlock (PeacockStats) │ df, save_path │ ['outpath']\n", + "CollectHNMFkLeafBlock (LeafDataLabels) │ df, save_path │ ['leaf_data_csv', 'leaf_labels_csv']\n", + "PipelineSummaryBlock (PipelineSummary) │ │ ['docs_summary_df', 'docs_summary_plot']\n", + "TermiteNeo4jBlock (TermiteNeo4j) │ leaf_data_csv, leaf_labels_csv │ ['data_triplets_csv', 'topic_triplets_csv']\n", + "TermiteVectorBlock (TermiteVectorIndex) │ leaf_data_csv │ ['vector_index_name', 'vector_stats']\n", + "\n" + ] + } + ], + "source": [ + "manager = BlockManager(\n", + " blocks = [\n", + " duplicate_cleaner_block,\n", + " vulture_block,\n", + " orca_block,\n", + " clean_affiliations_block,\n", + " vocab_block,\n", + " matrix_block,\n", + " hnmfk_block,\n", + " term_attribute_block,\n", + " wolf_coauthor_block,\n", + " wolf_coaffiliation_block, \n", + " post_process_block, \n", + " peacock_block,\n", + " collect_leaves,\n", + " summary_block,\n", + " neo4j_block,\n", + " vector_store_block\n", + " ],\n", + " databundle=bundle, \n", + " progress = True, # see which block is executing\n", + " capture_output=None #'file',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "d3ded72e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "▶ [1/15] 01_VultureClean …\n", + "[01_VultureClean] ✔ loaded from checkpoint\n", + "✓ [1/15] 01_VultureClean finished in 2.03s\n", + "▶ [2/15] 02_Orca …\n", + "[02_Orca] ✔ loaded from checkpoint\n", + "✓ [2/15] 02_Orca finished in 0.01s\n", + "▶ [3/15] 03_CleanAffiliations …\n", + "✓ [3/15] 03_CleanAffiliations finished in 0.00s\n", + "▶ [4/15] 04_BeaverVocab …\n", + "✓ [4/15] 04_BeaverVocab finished in 0.01s\n", + "▶ [5/15] 05_DocWord …\n", + "✓ [5/15] 05_DocWord finished in 0.01s\n", + "▶ [6/15] 06_SemanticHNMFk …\n", + "Continuing from checkpoint...\n", + "Loading saved object state from checkpoint...\n", + "Done\n", + "Loading saved object state from checkpoint...\n", + "[06_SemanticHNMFk] ⭳ checkpoint saved\n", + "✓ [6/15] 06_SemanticHNMFk finished in 0.01s\n", + "▶ [7/15] 07_Attribution …\n", + "[07_Attribution] ✔ loaded from checkpoint\n", + "✓ [7/15] 07_Attribution finished in 0.01s\n", + "▶ [8/15] 08_WolfAuthor …\n", + "[08_WolfAuthor] ✔ loaded from checkpoint\n", + "✓ [8/15] 08_WolfAuthor finished in 0.00s\n", + "▶ [9/15] 09_WolfAffil …\n", + "[09_WolfAffil] ✔ loaded from checkpoint\n", + "✓ [9/15] 09_WolfAffil finished in 0.00s\n", + "▶ [10/15] 10_ArticFox …\n", + "[10_ArticFox] ✔ loaded from checkpoint\n", + "✓ [10/15] 10_ArticFox finished in 0.00s\n", + "▶ [11/15] 11_PeacockStats …\n", + "[11_PeacockStats] ✔ loaded from checkpoint\n", + "✓ [11/15] 11_PeacockStats finished in 0.00s\n", + "▶ [12/15] 12_LeafDataLabels …\n", + "[12_LeafDataLabels] Wrote:\n", + " /Users/barron/Desktop/collections_telf_pipeline/telf_internal/examples/Full TELF Pipeline/single_block_examples/example_results/semantic_HNMFk_collection_slurm_option/12_LeafDataLabels/LEAF_DATA.csv\n", + " /Users/barron/Desktop/collections_telf_pipeline/telf_internal/examples/Full TELF Pipeline/single_block_examples/example_results/semantic_HNMFk_collection_slurm_option/12_LeafDataLabels/LEAF_LABELS.csv\n", + " /Users/barron/Desktop/collections_telf_pipeline/telf_internal/examples/Full TELF Pipeline/single_block_examples/example_results/semantic_HNMFk_collection_slurm_option/12_LeafDataLabels/summary.txt\n", + "[12_LeafDataLabels] ⭳ checkpoint saved\n", + "✓ [12/15] 12_LeafDataLabels finished in 0.07s\n", + "▶ [13/15] 13_PipelineSummary …\n", + "[13_PipelineSummary] ✔ loaded from checkpoint\n", + "✓ [13/15] 13_PipelineSummary finished in 0.02s\n", + "▶ [14/15] 14_TermiteNeo4j …\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Topic_ID_id_unique FOR (n:Topic_ID) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=4, name='Topic_ID_id_unique', type='UNIQUENESS', schema=(:Topic_ID {id}), ownedIndex=3 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Document_ID_id_unique FOR (n:Document_ID) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=6, name='Document_ID_id_unique', type='UNIQUENESS', schema=(:Document_ID {id}), ownedIndex=5 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Affiliation_ID_id_unique FOR (n:Affiliation_ID) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=8, name='Affiliation_ID_id_unique', type='UNIQUENESS', schema=(:Affiliation_ID {id}), ownedIndex=7 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Country_id_unique FOR (n:Country) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=10, name='Country_id_unique', type='UNIQUENESS', schema=(:Country {id}), ownedIndex=9 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Scopus_category_id_unique FOR (n:Scopus_category) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=12, name='Scopus_category_id_unique', type='UNIQUENESS', schema=(:Scopus_category {id}), ownedIndex=11 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT acronym_id_unique FOR (n:acronym) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=14, name='acronym_id_unique', type='UNIQUENESS', schema=(:acronym {id}), ownedIndex=13 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Year_id_unique FOR (n:Year) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=16, name='Year_id_unique', type='UNIQUENESS', schema=(:Year {id}), ownedIndex=15 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Author_ID_id_unique FOR (n:Author_ID) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=18, name='Author_ID_id_unique', type='UNIQUENESS', schema=(:Author_ID {id}), ownedIndex=17 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Publisher_id_unique FOR (n:Publisher) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=20, name='Publisher_id_unique', type='UNIQUENESS', schema=(:Publisher {id}), ownedIndex=19 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Topic_ID_id_unique FOR (n:Topic_ID) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=4, name='Topic_ID_id_unique', type='UNIQUENESS', schema=(:Topic_ID {id}), ownedIndex=3 )'.}\n", + "Failed to create unique constraint on: \n", + "\tCREATE CONSTRAINT Keyword_id_unique FOR (n:Keyword) REQUIRE n.id IS UNIQUE \n", + "\t {code: Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists} {message: An equivalent constraint already exists, 'Constraint( id=22, name='Keyword_id_unique', type='UNIQUENESS', schema=(:Keyword {id}), ownedIndex=21 )'.}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 42/42 [00:00<00:00, 2101.76it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_0_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1993, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Frontiers in Neural Computation', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_0_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')]}, 'Year': {'entity': 1993, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Frontiers in Neural Computation', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 'Root_0_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 1993, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 'Frontiers in Neural Computation', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_1_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf', 'weight': None, 'attributes': [('Title', 'Federated Learning and Privacy-Preserving AI Machine Learning for Edge Computing Applications Reinforcement Learning for Game AI'), ('EID', '6432d6ae-6269-4d2d-98a8-7f4baa587316'), ('S2ID', 'f0155ef4-dba8-4b86-9873-940585196862'), ('DOI', 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1992, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Proceedings of the Global AI Summit;Frontiers in Neural Computation', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_1_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf', 'weight': None, 'attributes': [('Title', 'Federated Learning and Privacy-Preserving AI Machine Learning for Edge Computing Applications Reinforcement Learning for Game AI'), ('EID', '6432d6ae-6269-4d2d-98a8-7f4baa587316'), ('S2ID', 'f0155ef4-dba8-4b86-9873-940585196862'), ('DOI', 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf')]}, 'Year': {'entity': 1992, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Proceedings of the Global AI Summit;Frontiers in Neural Computation', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf', 'tail': 'Root_1_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Federated Learning and Privacy-Preserving AI Machine Learning for Edge Computing Applications Reinforcement Learning for Game AI'), ('EID', '6432d6ae-6269-4d2d-98a8-7f4baa587316'), ('S2ID', 'f0155ef4-dba8-4b86-9873-940585196862'), ('DOI', 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf', 'tail': 1992, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Federated Learning and Privacy-Preserving AI Machine Learning for Edge Computing Applications Reinforcement Learning for Game AI'), ('EID', '6432d6ae-6269-4d2d-98a8-7f4baa587316'), ('S2ID', 'f0155ef4-dba8-4b86-9873-940585196862'), ('DOI', 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf', 'tail': 'Proceedings of the Global AI Summit;Frontiers in Neural Computation', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Federated Learning and Privacy-Preserving AI Machine Learning for Edge Computing Applications Reinforcement Learning for Game AI'), ('EID', '6432d6ae-6269-4d2d-98a8-7f4baa587316'), ('S2ID', 'f0155ef4-dba8-4b86-9873-940585196862'), ('DOI', 'a3d859c3-0c13-4c47-acbb-d3d87a9c7ecf')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_1_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '570e8a12-be16-4362-b815-73b3a6ca7bbd', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI Deep Reinforcement Learning for Robotics Exploring Deep Learning for Autonomous Systems'), ('EID', '703a7b27-ba5e-4fb7-9396-09bb0bf5d5c1'), ('S2ID', 'b4dcbb21-0358-4860-8019-a2b722f46dbd'), ('DOI', '570e8a12-be16-4362-b815-73b3a6ca7bbd')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2005, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Conference on Explainable and Trustworthy AI;Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_1_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '570e8a12-be16-4362-b815-73b3a6ca7bbd', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI Deep Reinforcement Learning for Robotics Exploring Deep Learning for Autonomous Systems'), ('EID', '703a7b27-ba5e-4fb7-9396-09bb0bf5d5c1'), ('S2ID', 'b4dcbb21-0358-4860-8019-a2b722f46dbd'), ('DOI', '570e8a12-be16-4362-b815-73b3a6ca7bbd')]}, 'Year': {'entity': 2005, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Conference on Explainable and Trustworthy AI;Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '570e8a12-be16-4362-b815-73b3a6ca7bbd', 'tail': 'Root_1_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI Deep Reinforcement Learning for Robotics Exploring Deep Learning for Autonomous Systems'), ('EID', '703a7b27-ba5e-4fb7-9396-09bb0bf5d5c1'), ('S2ID', 'b4dcbb21-0358-4860-8019-a2b722f46dbd'), ('DOI', '570e8a12-be16-4362-b815-73b3a6ca7bbd')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '570e8a12-be16-4362-b815-73b3a6ca7bbd', 'tail': 2005, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI Deep Reinforcement Learning for Robotics Exploring Deep Learning for Autonomous Systems'), ('EID', '703a7b27-ba5e-4fb7-9396-09bb0bf5d5c1'), ('S2ID', 'b4dcbb21-0358-4860-8019-a2b722f46dbd'), ('DOI', '570e8a12-be16-4362-b815-73b3a6ca7bbd')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '570e8a12-be16-4362-b815-73b3a6ca7bbd', 'tail': 'Conference on Explainable and Trustworthy AI;Symposium on AI for Sustainable Development', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI Deep Reinforcement Learning for Robotics Exploring Deep Learning for Autonomous Systems'), ('EID', '703a7b27-ba5e-4fb7-9396-09bb0bf5d5c1'), ('S2ID', 'b4dcbb21-0358-4860-8019-a2b722f46dbd'), ('DOI', '570e8a12-be16-4362-b815-73b3a6ca7bbd')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_1_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d', 'weight': None, 'attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models AutoML: The Future of Automated Data Science Dimensionality Reduction Techniques for Big Data'), ('EID', 'a052ecb8-b5e9-4b91-b075-66af46e4cd20'), ('S2ID', 'e4a5f4b6-b5e1-4509-8e51-9727ab526046'), ('DOI', '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2004, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'International Journal of Machine Intelligence;Annual Summit on Quantum Machine Learning', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_1_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d', 'weight': None, 'attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models AutoML: The Future of Automated Data Science Dimensionality Reduction Techniques for Big Data'), ('EID', 'a052ecb8-b5e9-4b91-b075-66af46e4cd20'), ('S2ID', 'e4a5f4b6-b5e1-4509-8e51-9727ab526046'), ('DOI', '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d')]}, 'Year': {'entity': 2004, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'International Journal of Machine Intelligence;Annual Summit on Quantum Machine Learning', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d', 'tail': 'Root_1_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models AutoML: The Future of Automated Data Science Dimensionality Reduction Techniques for Big Data'), ('EID', 'a052ecb8-b5e9-4b91-b075-66af46e4cd20'), ('S2ID', 'e4a5f4b6-b5e1-4509-8e51-9727ab526046'), ('DOI', '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d', 'tail': 2004, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models AutoML: The Future of Automated Data Science Dimensionality Reduction Techniques for Big Data'), ('EID', 'a052ecb8-b5e9-4b91-b075-66af46e4cd20'), ('S2ID', 'e4a5f4b6-b5e1-4509-8e51-9727ab526046'), ('DOI', '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d', 'tail': 'International Journal of Machine Intelligence;Annual Summit on Quantum Machine Learning', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models AutoML: The Future of Automated Data Science Dimensionality Reduction Techniques for Big Data'), ('EID', 'a052ecb8-b5e9-4b91-b075-66af46e4cd20'), ('S2ID', 'e4a5f4b6-b5e1-4509-8e51-9727ab526046'), ('DOI', '6dcde3fa-b28d-4ef6-a1ea-5489c323e49d')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_2_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2006, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'AI Research Frontiers Conference', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_2_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')]}, 'Year': {'entity': 2006, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'AI Research Frontiers Conference', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'tail': 'Root_2_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'tail': 2006, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'tail': 'AI Research Frontiers Conference', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_3_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Symposium on Computational AI and Ethics', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_3_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Symposium on Computational AI and Ethics', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 'Root_3_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 'Symposium on Computational AI and Ethics', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_3_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '080772ff-df90-40c9-8a35-35b5dda537d9', 'weight': None, 'attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2004, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Annual Conference on AI and Robotics', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_3_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '080772ff-df90-40c9-8a35-35b5dda537d9', 'weight': None, 'attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')]}, 'Year': {'entity': 2004, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Annual Conference on AI and Robotics', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '080772ff-df90-40c9-8a35-35b5dda537d9', 'tail': 'Root_3_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '080772ff-df90-40c9-8a35-35b5dda537d9', 'tail': 2004, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '080772ff-df90-40c9-8a35-35b5dda537d9', 'tail': 'Annual Conference on AI and Robotics', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_3_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2001, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Annual Summit on Quantum Machine Learning;Journal of Computational Vision and AI', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_3_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')]}, 'Year': {'entity': 2001, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Annual Summit on Quantum Machine Learning;Journal of Computational Vision and AI', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'tail': 'Root_3_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'tail': 2001, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'tail': 'Annual Summit on Quantum Machine Learning;Journal of Computational Vision and AI', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_3_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'e58bf863-7402-4fcb-9cac-58e3d8780872', 'weight': None, 'attributes': [('Title', 'Deep Reinforcement Learning for Robotics'), ('EID', '0121772d-5b67-4bf2-add1-38aa73403128'), ('S2ID', '452b7eb2-42f9-4afc-8669-a8b09e343127'), ('DOI', 'e58bf863-7402-4fcb-9cac-58e3d8780872')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2002, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Global AI and Big Data Innovations;Neural Computation and Learning Symposium', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_3_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'e58bf863-7402-4fcb-9cac-58e3d8780872', 'weight': None, 'attributes': [('Title', 'Deep Reinforcement Learning for Robotics'), ('EID', '0121772d-5b67-4bf2-add1-38aa73403128'), ('S2ID', '452b7eb2-42f9-4afc-8669-a8b09e343127'), ('DOI', 'e58bf863-7402-4fcb-9cac-58e3d8780872')]}, 'Year': {'entity': 2002, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Global AI and Big Data Innovations;Neural Computation and Learning Symposium', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e58bf863-7402-4fcb-9cac-58e3d8780872', 'tail': 'Root_3_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Deep Reinforcement Learning for Robotics'), ('EID', '0121772d-5b67-4bf2-add1-38aa73403128'), ('S2ID', '452b7eb2-42f9-4afc-8669-a8b09e343127'), ('DOI', 'e58bf863-7402-4fcb-9cac-58e3d8780872')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e58bf863-7402-4fcb-9cac-58e3d8780872', 'tail': 2002, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Deep Reinforcement Learning for Robotics'), ('EID', '0121772d-5b67-4bf2-add1-38aa73403128'), ('S2ID', '452b7eb2-42f9-4afc-8669-a8b09e343127'), ('DOI', 'e58bf863-7402-4fcb-9cac-58e3d8780872')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'c55a97d8-cf05-4014-b4d6-01730a79c2b4', 'weight': None, 'attributes': [('name', 'Laboratory of Apple Two Cedar')]}]\n", + "[{'entity': 'Four Birch', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e58bf863-7402-4fcb-9cac-58e3d8780872', 'tail': 'Global AI and Big Data Innovations;Neural Computation and Learning Symposium', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Deep Reinforcement Learning for Robotics'), ('EID', '0121772d-5b67-4bf2-add1-38aa73403128'), ('S2ID', '452b7eb2-42f9-4afc-8669-a8b09e343127'), ('DOI', 'e58bf863-7402-4fcb-9cac-58e3d8780872')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_3_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '633907a2-1684-4114-ba15-1718e787d011', 'weight': None, 'attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models Exploring Deep Learning for Autonomous Systems Synthetic Data Generation for Training AI Models'), ('EID', 'c6f47364-ef7c-4439-803f-2e8d3e781811'), ('S2ID', '8f062270-72e5-4e77-a343-4cd77ed3f255'), ('DOI', '633907a2-1684-4114-ba15-1718e787d011')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2002, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'International Workshop on Learning Algorithms', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_3_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '633907a2-1684-4114-ba15-1718e787d011', 'weight': None, 'attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models Exploring Deep Learning for Autonomous Systems Synthetic Data Generation for Training AI Models'), ('EID', 'c6f47364-ef7c-4439-803f-2e8d3e781811'), ('S2ID', '8f062270-72e5-4e77-a343-4cd77ed3f255'), ('DOI', '633907a2-1684-4114-ba15-1718e787d011')]}, 'Year': {'entity': 2002, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'International Workshop on Learning Algorithms', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '633907a2-1684-4114-ba15-1718e787d011', 'tail': 'Root_3_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models Exploring Deep Learning for Autonomous Systems Synthetic Data Generation for Training AI Models'), ('EID', 'c6f47364-ef7c-4439-803f-2e8d3e781811'), ('S2ID', '8f062270-72e5-4e77-a343-4cd77ed3f255'), ('DOI', '633907a2-1684-4114-ba15-1718e787d011')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '633907a2-1684-4114-ba15-1718e787d011', 'tail': 2002, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models Exploring Deep Learning for Autonomous Systems Synthetic Data Generation for Training AI Models'), ('EID', 'c6f47364-ef7c-4439-803f-2e8d3e781811'), ('S2ID', '8f062270-72e5-4e77-a343-4cd77ed3f255'), ('DOI', '633907a2-1684-4114-ba15-1718e787d011')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'c55a97d8-cf05-4014-b4d6-01730a79c2b4', 'weight': None, 'attributes': [('name', 'Laboratory of Apple Two Cedar')]}]\n", + "[{'entity': 'Four Birch', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '633907a2-1684-4114-ba15-1718e787d011', 'tail': 'International Workshop on Learning Algorithms', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Optimizing Hyperparameters in Large Language Models Exploring Deep Learning for Autonomous Systems Synthetic Data Generation for Training AI Models'), ('EID', 'c6f47364-ef7c-4439-803f-2e8d3e781811'), ('S2ID', '8f062270-72e5-4e77-a343-4cd77ed3f255'), ('DOI', '633907a2-1684-4114-ba15-1718e787d011')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_4_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'weight': None, 'attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1999, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_4_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'weight': None, 'attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')]}, 'Year': {'entity': 1999, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'tail': 'Root_4_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'tail': 1999, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': '7621ffa5-659c-4278-9237-3286f1bad64e', 'weight': None, 'attributes': [('name', 'Organization of Cucumber Grapes')]}]\n", + "[{'entity': 'Elm Willow', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'tail': 'Symposium on AI for Sustainable Development', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_4_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'weight': None, 'attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1994, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Global Summit on AI-Driven Technologies', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_4_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'weight': None, 'attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')]}, 'Year': {'entity': 1994, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Global Summit on AI-Driven Technologies', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'tail': 'Root_4_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'tail': 1994, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'tail': 'Global Summit on AI-Driven Technologies', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_4_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '99b8525e-16b5-4403-b113-e9328ae9b4f4', 'weight': None, 'attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities Multi-Modal Learning for Speech and Vision Multi-Modal Learning for Speech and Vision'), ('EID', '2d3f5f05-e6d5-4968-8868-cfb6a7cbf46f'), ('S2ID', '86643ebe-4356-4425-a3f6-c6723f7b9f04'), ('DOI', '99b8525e-16b5-4403-b113-e9328ae9b4f4')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1994, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Journal of Intelligent Systems and Automation', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_4_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '99b8525e-16b5-4403-b113-e9328ae9b4f4', 'weight': None, 'attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities Multi-Modal Learning for Speech and Vision Multi-Modal Learning for Speech and Vision'), ('EID', '2d3f5f05-e6d5-4968-8868-cfb6a7cbf46f'), ('S2ID', '86643ebe-4356-4425-a3f6-c6723f7b9f04'), ('DOI', '99b8525e-16b5-4403-b113-e9328ae9b4f4')]}, 'Year': {'entity': 1994, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Journal of Intelligent Systems and Automation', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '99b8525e-16b5-4403-b113-e9328ae9b4f4', 'tail': 'Root_4_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities Multi-Modal Learning for Speech and Vision Multi-Modal Learning for Speech and Vision'), ('EID', '2d3f5f05-e6d5-4968-8868-cfb6a7cbf46f'), ('S2ID', '86643ebe-4356-4425-a3f6-c6723f7b9f04'), ('DOI', '99b8525e-16b5-4403-b113-e9328ae9b4f4')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '99b8525e-16b5-4403-b113-e9328ae9b4f4', 'tail': 1994, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities Multi-Modal Learning for Speech and Vision Multi-Modal Learning for Speech and Vision'), ('EID', '2d3f5f05-e6d5-4968-8868-cfb6a7cbf46f'), ('S2ID', '86643ebe-4356-4425-a3f6-c6723f7b9f04'), ('DOI', '99b8525e-16b5-4403-b113-e9328ae9b4f4')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '99b8525e-16b5-4403-b113-e9328ae9b4f4', 'tail': 'Journal of Intelligent Systems and Automation', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities Multi-Modal Learning for Speech and Vision Multi-Modal Learning for Speech and Vision'), ('EID', '2d3f5f05-e6d5-4968-8868-cfb6a7cbf46f'), ('S2ID', '86643ebe-4356-4425-a3f6-c6723f7b9f04'), ('DOI', '99b8525e-16b5-4403-b113-e9328ae9b4f4')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_5_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1993, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Frontiers in Neural Computation', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_5_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')]}, 'Year': {'entity': 1993, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Frontiers in Neural Computation', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 'Root_5_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 1993, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 'Frontiers in Neural Computation', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_5_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '7133638d-7ec8-4c4b-b048-78105da4eace', 'weight': None, 'attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2001, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Frontiers in Neural Computation;Journal of Deep Learning Innovations', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_5_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '7133638d-7ec8-4c4b-b048-78105da4eace', 'weight': None, 'attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')]}, 'Year': {'entity': 2001, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Frontiers in Neural Computation;Journal of Deep Learning Innovations', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7133638d-7ec8-4c4b-b048-78105da4eace', 'tail': 'Root_5_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7133638d-7ec8-4c4b-b048-78105da4eace', 'tail': 2001, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7133638d-7ec8-4c4b-b048-78105da4eace', 'tail': 'Frontiers in Neural Computation;Journal of Deep Learning Innovations', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_5_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '080772ff-df90-40c9-8a35-35b5dda537d9', 'weight': None, 'attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2004, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Annual Conference on AI and Robotics', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_5_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '080772ff-df90-40c9-8a35-35b5dda537d9', 'weight': None, 'attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')]}, 'Year': {'entity': 2004, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Annual Conference on AI and Robotics', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '080772ff-df90-40c9-8a35-35b5dda537d9', 'tail': 'Root_5_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '080772ff-df90-40c9-8a35-35b5dda537d9', 'tail': 2004, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '080772ff-df90-40c9-8a35-35b5dda537d9', 'tail': 'Annual Conference on AI and Robotics', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'The Future of AI in Predictive Analytics'), ('EID', '38279269-ebe8-4b16-a758-322f76fe2ea1'), ('S2ID', '42ee72f1-eda5-4c67-aced-cc9b00175366'), ('DOI', '080772ff-df90-40c9-8a35-35b5dda537d9')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_5_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '7b50669f-5148-4da5-a26b-17196b7975e6', 'weight': None, 'attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'International Journal of Machine Intelligence', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_5_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '7b50669f-5148-4da5-a26b-17196b7975e6', 'weight': None, 'attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'International Journal of Machine Intelligence', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7b50669f-5148-4da5-a26b-17196b7975e6', 'tail': 'Root_5_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7b50669f-5148-4da5-a26b-17196b7975e6', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7b50669f-5148-4da5-a26b-17196b7975e6', 'tail': 'International Journal of Machine Intelligence', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_5_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'aa7352e5-be55-40b3-9228-e8e05745a66c', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI The Role of Generative AI in Creative Industries'), ('EID', 'ab5bdd50-a00d-4407-b337-b4c75b226f1f'), ('S2ID', '78b9bb6b-ca26-4cc5-8374-1d04b001298b'), ('DOI', 'aa7352e5-be55-40b3-9228-e8e05745a66c')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2005, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Journal of Deep Learning Innovations;AI Research Frontiers Conference', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_5_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'aa7352e5-be55-40b3-9228-e8e05745a66c', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI The Role of Generative AI in Creative Industries'), ('EID', 'ab5bdd50-a00d-4407-b337-b4c75b226f1f'), ('S2ID', '78b9bb6b-ca26-4cc5-8374-1d04b001298b'), ('DOI', 'aa7352e5-be55-40b3-9228-e8e05745a66c')]}, 'Year': {'entity': 2005, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Journal of Deep Learning Innovations;AI Research Frontiers Conference', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'aa7352e5-be55-40b3-9228-e8e05745a66c', 'tail': 'Root_5_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI The Role of Generative AI in Creative Industries'), ('EID', 'ab5bdd50-a00d-4407-b337-b4c75b226f1f'), ('S2ID', '78b9bb6b-ca26-4cc5-8374-1d04b001298b'), ('DOI', 'aa7352e5-be55-40b3-9228-e8e05745a66c')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'aa7352e5-be55-40b3-9228-e8e05745a66c', 'tail': 2005, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI The Role of Generative AI in Creative Industries'), ('EID', 'ab5bdd50-a00d-4407-b337-b4c75b226f1f'), ('S2ID', '78b9bb6b-ca26-4cc5-8374-1d04b001298b'), ('DOI', 'aa7352e5-be55-40b3-9228-e8e05745a66c')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'aa7352e5-be55-40b3-9228-e8e05745a66c', 'tail': 'Journal of Deep Learning Innovations;AI Research Frontiers Conference', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI The Role of Generative AI in Creative Industries'), ('EID', 'ab5bdd50-a00d-4407-b337-b4c75b226f1f'), ('S2ID', '78b9bb6b-ca26-4cc5-8374-1d04b001298b'), ('DOI', 'aa7352e5-be55-40b3-9228-e8e05745a66c')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_6_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Symposium on Computational AI and Ethics', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_6_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Symposium on Computational AI and Ethics', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 'Root_6_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 'Symposium on Computational AI and Ethics', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_6_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1993, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Frontiers in Neural Computation', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_6_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')]}, 'Year': {'entity': 1993, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Frontiers in Neural Computation', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 'Root_6_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 1993, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '8b37f74e-ec68-44fe-88a9-5830cdf7ea48', 'tail': 'Frontiers in Neural Computation', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '4b230c89-61aa-4b61-8780-87c12fbf9183'), ('S2ID', '43245345-6e05-4a65-96fe-9b8bd15bb9ad'), ('DOI', '8b37f74e-ec68-44fe-88a9-5830cdf7ea48')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_6_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'f03c84ba-ac61-429b-80aa-96e1eb023cdd', 'weight': None, 'attributes': [('Title', 'Building Robust AI Models with Adversarial Training Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis'), ('EID', 'fc2cd2f7-2431-472b-9f23-aa9afb4ccf7f'), ('S2ID', 'f49acf4d-1287-499e-bb1f-a571a7b4caa7'), ('DOI', 'f03c84ba-ac61-429b-80aa-96e1eb023cdd')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1996, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Proceedings of the Global AI Summit', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_6_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'f03c84ba-ac61-429b-80aa-96e1eb023cdd', 'weight': None, 'attributes': [('Title', 'Building Robust AI Models with Adversarial Training Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis'), ('EID', 'fc2cd2f7-2431-472b-9f23-aa9afb4ccf7f'), ('S2ID', 'f49acf4d-1287-499e-bb1f-a571a7b4caa7'), ('DOI', 'f03c84ba-ac61-429b-80aa-96e1eb023cdd')]}, 'Year': {'entity': 1996, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Proceedings of the Global AI Summit', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f03c84ba-ac61-429b-80aa-96e1eb023cdd', 'tail': 'Root_6_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis'), ('EID', 'fc2cd2f7-2431-472b-9f23-aa9afb4ccf7f'), ('S2ID', 'f49acf4d-1287-499e-bb1f-a571a7b4caa7'), ('DOI', 'f03c84ba-ac61-429b-80aa-96e1eb023cdd')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f03c84ba-ac61-429b-80aa-96e1eb023cdd', 'tail': 1996, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis'), ('EID', 'fc2cd2f7-2431-472b-9f23-aa9afb4ccf7f'), ('S2ID', 'f49acf4d-1287-499e-bb1f-a571a7b4caa7'), ('DOI', 'f03c84ba-ac61-429b-80aa-96e1eb023cdd')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f03c84ba-ac61-429b-80aa-96e1eb023cdd', 'tail': 'Proceedings of the Global AI Summit', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis'), ('EID', 'fc2cd2f7-2431-472b-9f23-aa9afb4ccf7f'), ('S2ID', 'f49acf4d-1287-499e-bb1f-a571a7b4caa7'), ('DOI', 'f03c84ba-ac61-429b-80aa-96e1eb023cdd')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_6_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '5ba3a719-27eb-48c5-a072-372903a106c2', 'weight': None, 'attributes': [('Title', 'How Transformers are Revolutionizing NLP Federated Learning and Privacy-Preserving AI Synthetic Data Generation for Training AI Models'), ('EID', '4f3976e3-c1fb-44fb-97f4-8783bd1aca8c'), ('S2ID', '4cde7440-280b-48aa-b2a2-4e28a59f72a3'), ('DOI', '5ba3a719-27eb-48c5-a072-372903a106c2')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2000, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'AI and Cybersecurity Research Symposium;Next-Gen Neural Networks Workshop', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_6_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '5ba3a719-27eb-48c5-a072-372903a106c2', 'weight': None, 'attributes': [('Title', 'How Transformers are Revolutionizing NLP Federated Learning and Privacy-Preserving AI Synthetic Data Generation for Training AI Models'), ('EID', '4f3976e3-c1fb-44fb-97f4-8783bd1aca8c'), ('S2ID', '4cde7440-280b-48aa-b2a2-4e28a59f72a3'), ('DOI', '5ba3a719-27eb-48c5-a072-372903a106c2')]}, 'Year': {'entity': 2000, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'AI and Cybersecurity Research Symposium;Next-Gen Neural Networks Workshop', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '5ba3a719-27eb-48c5-a072-372903a106c2', 'tail': 'Root_6_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'How Transformers are Revolutionizing NLP Federated Learning and Privacy-Preserving AI Synthetic Data Generation for Training AI Models'), ('EID', '4f3976e3-c1fb-44fb-97f4-8783bd1aca8c'), ('S2ID', '4cde7440-280b-48aa-b2a2-4e28a59f72a3'), ('DOI', '5ba3a719-27eb-48c5-a072-372903a106c2')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '5ba3a719-27eb-48c5-a072-372903a106c2', 'tail': 2000, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'How Transformers are Revolutionizing NLP Federated Learning and Privacy-Preserving AI Synthetic Data Generation for Training AI Models'), ('EID', '4f3976e3-c1fb-44fb-97f4-8783bd1aca8c'), ('S2ID', '4cde7440-280b-48aa-b2a2-4e28a59f72a3'), ('DOI', '5ba3a719-27eb-48c5-a072-372903a106c2')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': '7621ffa5-659c-4278-9237-3286f1bad64e', 'weight': None, 'attributes': [('name', 'Organization of Cucumber Grapes')]}]\n", + "[{'entity': 'Elm Willow', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '5ba3a719-27eb-48c5-a072-372903a106c2', 'tail': 'AI and Cybersecurity Research Symposium;Next-Gen Neural Networks Workshop', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'How Transformers are Revolutionizing NLP Federated Learning and Privacy-Preserving AI Synthetic Data Generation for Training AI Models'), ('EID', '4f3976e3-c1fb-44fb-97f4-8783bd1aca8c'), ('S2ID', '4cde7440-280b-48aa-b2a2-4e28a59f72a3'), ('DOI', '5ba3a719-27eb-48c5-a072-372903a106c2')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '12a243c3-bf37-4dab-b908-024b9af5bcfa'), ('S2ID', '83cc1ff7-6f04-4676-8ad4-d26f6906be64'), ('DOI', '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2006, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Journal of Deep Learning Innovations', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '12a243c3-bf37-4dab-b908-024b9af5bcfa'), ('S2ID', '83cc1ff7-6f04-4676-8ad4-d26f6906be64'), ('DOI', '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f')]}, 'Year': {'entity': 2006, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Journal of Deep Learning Innovations', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '12a243c3-bf37-4dab-b908-024b9af5bcfa'), ('S2ID', '83cc1ff7-6f04-4676-8ad4-d26f6906be64'), ('DOI', '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f', 'tail': 2006, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '12a243c3-bf37-4dab-b908-024b9af5bcfa'), ('S2ID', '83cc1ff7-6f04-4676-8ad4-d26f6906be64'), ('DOI', '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f', 'tail': 'Journal of Deep Learning Innovations', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '12a243c3-bf37-4dab-b908-024b9af5bcfa'), ('S2ID', '83cc1ff7-6f04-4676-8ad4-d26f6906be64'), ('DOI', '6ee5121e-9b1b-483d-bc15-0bfdbc0e7c4f')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2006, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'AI Research Frontiers Conference', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'weight': None, 'attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')]}, 'Year': {'entity': 2006, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'AI Research Frontiers Conference', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'tail': 2006, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e83ae162-327e-4c5c-a571-afbcad1a2e70', 'tail': 'AI Research Frontiers Conference', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Reinforcement Learning for Game AI AI-Powered Medical Diagnosis Systems'), ('EID', '4f64abbf-d3cf-4bcf-8422-61ee8584a730'), ('S2ID', '8dac6c5a-6d0a-48a3-830a-2ee300cb3140'), ('DOI', 'e83ae162-327e-4c5c-a571-afbcad1a2e70')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'a283d0e1-a87a-4b15-b316-97299db7688b', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '75f0e965-61ac-4d3e-9a8b-e133f6fc697e'), ('S2ID', '8816e9cc-e822-4318-ac7a-1bdca5caec30'), ('DOI', 'a283d0e1-a87a-4b15-b316-97299db7688b')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1998, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Conference on Explainable and Trustworthy AI;Machine Learning Innovations & Applications Journal', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'a283d0e1-a87a-4b15-b316-97299db7688b', 'weight': None, 'attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '75f0e965-61ac-4d3e-9a8b-e133f6fc697e'), ('S2ID', '8816e9cc-e822-4318-ac7a-1bdca5caec30'), ('DOI', 'a283d0e1-a87a-4b15-b316-97299db7688b')]}, 'Year': {'entity': 1998, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Conference on Explainable and Trustworthy AI;Machine Learning Innovations & Applications Journal', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'a283d0e1-a87a-4b15-b316-97299db7688b', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '75f0e965-61ac-4d3e-9a8b-e133f6fc697e'), ('S2ID', '8816e9cc-e822-4318-ac7a-1bdca5caec30'), ('DOI', 'a283d0e1-a87a-4b15-b316-97299db7688b')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'a283d0e1-a87a-4b15-b316-97299db7688b', 'tail': 1998, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '75f0e965-61ac-4d3e-9a8b-e133f6fc697e'), ('S2ID', '8816e9cc-e822-4318-ac7a-1bdca5caec30'), ('DOI', 'a283d0e1-a87a-4b15-b316-97299db7688b')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'c55a97d8-cf05-4014-b4d6-01730a79c2b4', 'weight': None, 'attributes': [('name', 'Laboratory of Apple Two Cedar')]}]\n", + "[{'entity': 'Four Birch', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'a283d0e1-a87a-4b15-b316-97299db7688b', 'tail': 'Conference on Explainable and Trustworthy AI;Machine Learning Innovations & Applications Journal', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'AI-Driven Forecasting Models in Finance'), ('EID', '75f0e965-61ac-4d3e-9a8b-e133f6fc697e'), ('S2ID', '8816e9cc-e822-4318-ac7a-1bdca5caec30'), ('DOI', 'a283d0e1-a87a-4b15-b316-97299db7688b')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5', 'weight': None, 'attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '2d90d690-3b1c-4a7b-8e01-557318499997'), ('S2ID', '8f4c340f-2baf-4c3d-87b9-b95f7f1a5029'), ('DOI', 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2001, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Conference on Explainable and Trustworthy AI;Proceedings of the Global AI Summit', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5', 'weight': None, 'attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '2d90d690-3b1c-4a7b-8e01-557318499997'), ('S2ID', '8f4c340f-2baf-4c3d-87b9-b95f7f1a5029'), ('DOI', 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5')]}, 'Year': {'entity': 2001, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Conference on Explainable and Trustworthy AI;Proceedings of the Global AI Summit', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '2d90d690-3b1c-4a7b-8e01-557318499997'), ('S2ID', '8f4c340f-2baf-4c3d-87b9-b95f7f1a5029'), ('DOI', 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5', 'tail': 2001, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '2d90d690-3b1c-4a7b-8e01-557318499997'), ('S2ID', '8f4c340f-2baf-4c3d-87b9-b95f7f1a5029'), ('DOI', 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'c55a97d8-cf05-4014-b4d6-01730a79c2b4', 'weight': None, 'attributes': [('name', 'Laboratory of Apple Two Cedar')]}]\n", + "[{'entity': 'Four Birch', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5', 'tail': 'Conference on Explainable and Trustworthy AI;Proceedings of the Global AI Summit', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '2d90d690-3b1c-4a7b-8e01-557318499997'), ('S2ID', '8f4c340f-2baf-4c3d-87b9-b95f7f1a5029'), ('DOI', 'd9cb611c-8a26-45b1-88b0-5c0f0abaa8d5')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Symposium on Computational AI and Ethics', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Symposium on Computational AI and Ethics', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'd984689d-9e65-43ae-a60e-a22f31c39358', 'tail': 'Symposium on Computational AI and Ethics', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI Exploring Deep Learning for Autonomous Systems Sentiment Analysis with Large Language Models'), ('EID', '4df6a9cd-c60f-4d46-a30d-d7c9b0bf0bea'), ('S2ID', '0f4c886a-33c4-483f-ad03-8274eecd9138'), ('DOI', 'd984689d-9e65-43ae-a60e-a22f31c39358')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'de906aa7-7633-4193-be7e-bf5bf55e004d', 'weight': None, 'attributes': [('Title', 'Neural Architecture Search for Optimized Model Design'), ('EID', '67f9df07-c1b5-4d55-8a2c-6b85b184e54b'), ('S2ID', '3fbaca54-36ab-4209-af61-6d4ac0ec1194'), ('DOI', 'de906aa7-7633-4193-be7e-bf5bf55e004d')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1996, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'de906aa7-7633-4193-be7e-bf5bf55e004d', 'weight': None, 'attributes': [('Title', 'Neural Architecture Search for Optimized Model Design'), ('EID', '67f9df07-c1b5-4d55-8a2c-6b85b184e54b'), ('S2ID', '3fbaca54-36ab-4209-af61-6d4ac0ec1194'), ('DOI', 'de906aa7-7633-4193-be7e-bf5bf55e004d')]}, 'Year': {'entity': 1996, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'de906aa7-7633-4193-be7e-bf5bf55e004d', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Neural Architecture Search for Optimized Model Design'), ('EID', '67f9df07-c1b5-4d55-8a2c-6b85b184e54b'), ('S2ID', '3fbaca54-36ab-4209-af61-6d4ac0ec1194'), ('DOI', 'de906aa7-7633-4193-be7e-bf5bf55e004d')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'de906aa7-7633-4193-be7e-bf5bf55e004d', 'tail': 1996, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Neural Architecture Search for Optimized Model Design'), ('EID', '67f9df07-c1b5-4d55-8a2c-6b85b184e54b'), ('S2ID', '3fbaca54-36ab-4209-af61-6d4ac0ec1194'), ('DOI', 'de906aa7-7633-4193-be7e-bf5bf55e004d')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'de906aa7-7633-4193-be7e-bf5bf55e004d', 'tail': 'Symposium on AI for Sustainable Development', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Neural Architecture Search for Optimized Model Design'), ('EID', '67f9df07-c1b5-4d55-8a2c-6b85b184e54b'), ('S2ID', '3fbaca54-36ab-4209-af61-6d4ac0ec1194'), ('DOI', 'de906aa7-7633-4193-be7e-bf5bf55e004d')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_7_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'f432e7b3-8957-4b3d-b068-aaa76aed2e12', 'weight': None, 'attributes': [('Title', 'Deep Reinforcement Learning for Robotics Cybersecurity Threat Detection Using AI Bayesian Optimization in Hyperparameter Tuning'), ('EID', '3c224bfe-39ba-451a-b656-b09da35ada41'), ('S2ID', '9c26c2d5-c2e6-4eea-a125-75a08693754d'), ('DOI', 'f432e7b3-8957-4b3d-b068-aaa76aed2e12')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1996, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'International Journal of Machine Intelligence;Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_7_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'f432e7b3-8957-4b3d-b068-aaa76aed2e12', 'weight': None, 'attributes': [('Title', 'Deep Reinforcement Learning for Robotics Cybersecurity Threat Detection Using AI Bayesian Optimization in Hyperparameter Tuning'), ('EID', '3c224bfe-39ba-451a-b656-b09da35ada41'), ('S2ID', '9c26c2d5-c2e6-4eea-a125-75a08693754d'), ('DOI', 'f432e7b3-8957-4b3d-b068-aaa76aed2e12')]}, 'Year': {'entity': 1996, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'International Journal of Machine Intelligence;Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f432e7b3-8957-4b3d-b068-aaa76aed2e12', 'tail': 'Root_7_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Deep Reinforcement Learning for Robotics Cybersecurity Threat Detection Using AI Bayesian Optimization in Hyperparameter Tuning'), ('EID', '3c224bfe-39ba-451a-b656-b09da35ada41'), ('S2ID', '9c26c2d5-c2e6-4eea-a125-75a08693754d'), ('DOI', 'f432e7b3-8957-4b3d-b068-aaa76aed2e12')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f432e7b3-8957-4b3d-b068-aaa76aed2e12', 'tail': 1996, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Deep Reinforcement Learning for Robotics Cybersecurity Threat Detection Using AI Bayesian Optimization in Hyperparameter Tuning'), ('EID', '3c224bfe-39ba-451a-b656-b09da35ada41'), ('S2ID', '9c26c2d5-c2e6-4eea-a125-75a08693754d'), ('DOI', 'f432e7b3-8957-4b3d-b068-aaa76aed2e12')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'fcb9d5e2-a57f-45fa-b650-4a1a1e3d11fd', 'weight': None, 'attributes': [('name', 'Organization of Wolf Pasta Blue Oak')]}, {'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'Maroon Eight', 'weight': None, 'attributes': None}, {'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =2, tails_len=2\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f432e7b3-8957-4b3d-b068-aaa76aed2e12', 'tail': 'International Journal of Machine Intelligence;Symposium on AI for Sustainable Development', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Deep Reinforcement Learning for Robotics Cybersecurity Threat Detection Using AI Bayesian Optimization in Hyperparameter Tuning'), ('EID', '3c224bfe-39ba-451a-b656-b09da35ada41'), ('S2ID', '9c26c2d5-c2e6-4eea-a125-75a08693754d'), ('DOI', 'f432e7b3-8957-4b3d-b068-aaa76aed2e12')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '7b50669f-5148-4da5-a26b-17196b7975e6', 'weight': None, 'attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'International Journal of Machine Intelligence', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '7b50669f-5148-4da5-a26b-17196b7975e6', 'weight': None, 'attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'International Journal of Machine Intelligence', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7b50669f-5148-4da5-a26b-17196b7975e6', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7b50669f-5148-4da5-a26b-17196b7975e6', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7b50669f-5148-4da5-a26b-17196b7975e6', 'tail': 'International Journal of Machine Intelligence', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Using GANs for Realistic Image Synthesis Using GANs for Realistic Image Synthesis The Future of AI in Predictive Analytics'), ('EID', '50f5235f-b3d1-4243-9237-c37b12abcbd9'), ('S2ID', '6966b819-07f7-48fe-a9d9-12e72bb9436b'), ('DOI', '7b50669f-5148-4da5-a26b-17196b7975e6')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'weight': None, 'attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1994, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Global Summit on AI-Driven Technologies', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'weight': None, 'attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')]}, 'Year': {'entity': 1994, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Global Summit on AI-Driven Technologies', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'tail': 1994, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'e066a64b-d431-40b6-b95c-602f1aad83bc', 'tail': 'Global Summit on AI-Driven Technologies', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'A Comparative Study of CNNs and RNNs'), ('EID', '3a7681f7-5db9-4de8-a0b9-8e289369f83f'), ('S2ID', '83011446-5534-4e90-81f6-b72d4e25bf44'), ('DOI', 'e066a64b-d431-40b6-b95c-602f1aad83bc')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2001, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Annual Summit on Quantum Machine Learning;Journal of Computational Vision and AI', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')]}, 'Year': {'entity': 2001, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Annual Summit on Quantum Machine Learning;Journal of Computational Vision and AI', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'tail': 2001, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c', 'tail': 'Annual Summit on Quantum Machine Learning;Journal of Computational Vision and AI', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI'), ('EID', '8bd1dc3d-421a-441d-9fa4-21e624d24ced'), ('S2ID', 'e09fc743-d65b-4de1-b123-0d5c2b4cdc73'), ('DOI', '1ce36fd1-e394-4e5d-bf45-567ae88b3b8c')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '76d4bc7b-e459-4ce4-bf61-c953024a6431', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '77efd842-d5c5-4787-bd7d-ae5bcd48c71b'), ('S2ID', 'faba88ab-8044-4702-b1ea-bb9a57833370'), ('DOI', '76d4bc7b-e459-4ce4-bf61-c953024a6431')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2002, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'International Conference on Autonomous Learning;Annual Conference on AI and Robotics', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '76d4bc7b-e459-4ce4-bf61-c953024a6431', 'weight': None, 'attributes': [('Title', 'Unraveling the Mystery of Black Box AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '77efd842-d5c5-4787-bd7d-ae5bcd48c71b'), ('S2ID', 'faba88ab-8044-4702-b1ea-bb9a57833370'), ('DOI', '76d4bc7b-e459-4ce4-bf61-c953024a6431')]}, 'Year': {'entity': 2002, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'International Conference on Autonomous Learning;Annual Conference on AI and Robotics', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '76d4bc7b-e459-4ce4-bf61-c953024a6431', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '77efd842-d5c5-4787-bd7d-ae5bcd48c71b'), ('S2ID', 'faba88ab-8044-4702-b1ea-bb9a57833370'), ('DOI', '76d4bc7b-e459-4ce4-bf61-c953024a6431')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '76d4bc7b-e459-4ce4-bf61-c953024a6431', 'tail': 2002, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '77efd842-d5c5-4787-bd7d-ae5bcd48c71b'), ('S2ID', 'faba88ab-8044-4702-b1ea-bb9a57833370'), ('DOI', '76d4bc7b-e459-4ce4-bf61-c953024a6431')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'c55a97d8-cf05-4014-b4d6-01730a79c2b4', 'weight': None, 'attributes': [('name', 'Laboratory of Apple Two Cedar')]}]\n", + "[{'entity': 'Four Birch', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '76d4bc7b-e459-4ce4-bf61-c953024a6431', 'tail': 'International Conference on Autonomous Learning;Annual Conference on AI and Robotics', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Unraveling the Mystery of Black Box AI AI-Powered Recommendation Systems in E-Commerce'), ('EID', '77efd842-d5c5-4787-bd7d-ae5bcd48c71b'), ('S2ID', 'faba88ab-8044-4702-b1ea-bb9a57833370'), ('DOI', '76d4bc7b-e459-4ce4-bf61-c953024a6431')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '7133638d-7ec8-4c4b-b048-78105da4eace', 'weight': None, 'attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2001, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Frontiers in Neural Computation;Journal of Deep Learning Innovations', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '7133638d-7ec8-4c4b-b048-78105da4eace', 'weight': None, 'attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')]}, 'Year': {'entity': 2001, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Frontiers in Neural Computation;Journal of Deep Learning Innovations', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7133638d-7ec8-4c4b-b048-78105da4eace', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7133638d-7ec8-4c4b-b048-78105da4eace', 'tail': 2001, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7133638d-7ec8-4c4b-b048-78105da4eace', 'tail': 'Frontiers in Neural Computation;Journal of Deep Learning Innovations', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'The Role of Generative AI in Creative Industries'), ('EID', 'a98fb8ca-e775-473c-8ff1-022968c6ceec'), ('S2ID', '04982157-9c28-4a09-b0e4-98219d231e02'), ('DOI', '7133638d-7ec8-4c4b-b048-78105da4eace')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '7ee16f1c-f419-4d9d-9626-a30caa7459dd', 'weight': None, 'attributes': [('Title', 'AI-Powered Recommendation Systems in E-Commerce The Future of AI in Predictive Analytics'), ('EID', 'f61a999a-5a6d-4dd2-bc3c-b7128a54cee2'), ('S2ID', 'd5688547-a610-4c0f-b60f-62d2aa3ed4fc'), ('DOI', '7ee16f1c-f419-4d9d-9626-a30caa7459dd')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1994, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'AI Systems and Optimization Journal', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '7ee16f1c-f419-4d9d-9626-a30caa7459dd', 'weight': None, 'attributes': [('Title', 'AI-Powered Recommendation Systems in E-Commerce The Future of AI in Predictive Analytics'), ('EID', 'f61a999a-5a6d-4dd2-bc3c-b7128a54cee2'), ('S2ID', 'd5688547-a610-4c0f-b60f-62d2aa3ed4fc'), ('DOI', '7ee16f1c-f419-4d9d-9626-a30caa7459dd')]}, 'Year': {'entity': 1994, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'AI Systems and Optimization Journal', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7ee16f1c-f419-4d9d-9626-a30caa7459dd', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'AI-Powered Recommendation Systems in E-Commerce The Future of AI in Predictive Analytics'), ('EID', 'f61a999a-5a6d-4dd2-bc3c-b7128a54cee2'), ('S2ID', 'd5688547-a610-4c0f-b60f-62d2aa3ed4fc'), ('DOI', '7ee16f1c-f419-4d9d-9626-a30caa7459dd')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7ee16f1c-f419-4d9d-9626-a30caa7459dd', 'tail': 1994, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'AI-Powered Recommendation Systems in E-Commerce The Future of AI in Predictive Analytics'), ('EID', 'f61a999a-5a6d-4dd2-bc3c-b7128a54cee2'), ('S2ID', 'd5688547-a610-4c0f-b60f-62d2aa3ed4fc'), ('DOI', '7ee16f1c-f419-4d9d-9626-a30caa7459dd')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '7ee16f1c-f419-4d9d-9626-a30caa7459dd', 'tail': 'AI Systems and Optimization Journal', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'AI-Powered Recommendation Systems in E-Commerce The Future of AI in Predictive Analytics'), ('EID', 'f61a999a-5a6d-4dd2-bc3c-b7128a54cee2'), ('S2ID', 'd5688547-a610-4c0f-b60f-62d2aa3ed4fc'), ('DOI', '7ee16f1c-f419-4d9d-9626-a30caa7459dd')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'weight': None, 'attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1999, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'weight': None, 'attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')]}, 'Year': {'entity': 1999, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Symposium on AI for Sustainable Development', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'tail': 1999, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': '7621ffa5-659c-4278-9237-3286f1bad64e', 'weight': None, 'attributes': [('name', 'Organization of Cucumber Grapes')]}]\n", + "[{'entity': 'Elm Willow', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4', 'tail': 'Symposium on AI for Sustainable Development', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Building Robust AI Models with Adversarial Training'), ('EID', 'afc3db5e-f786-4cc3-8f15-b56d7c92420d'), ('S2ID', 'e64509da-9ba0-4f58-87d1-1fa48d74af48'), ('DOI', 'fce96ea0-f0fe-4388-96b9-03652bf3f9d4')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '59a2f67b-201f-4ad3-83d1-2895315f91a1', 'weight': None, 'attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI'), ('EID', 'c3860dc4-c5a3-46ff-866d-764140797a30'), ('S2ID', '606eff1d-3d86-416e-9fe8-69af7beb911e'), ('DOI', '59a2f67b-201f-4ad3-83d1-2895315f91a1')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1993, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Artificial Intelligence & Data Science Conference;Journal of Reinforcement Learning Strategies', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '59a2f67b-201f-4ad3-83d1-2895315f91a1', 'weight': None, 'attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI'), ('EID', 'c3860dc4-c5a3-46ff-866d-764140797a30'), ('S2ID', '606eff1d-3d86-416e-9fe8-69af7beb911e'), ('DOI', '59a2f67b-201f-4ad3-83d1-2895315f91a1')]}, 'Year': {'entity': 1993, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Artificial Intelligence & Data Science Conference;Journal of Reinforcement Learning Strategies', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '59a2f67b-201f-4ad3-83d1-2895315f91a1', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI'), ('EID', 'c3860dc4-c5a3-46ff-866d-764140797a30'), ('S2ID', '606eff1d-3d86-416e-9fe8-69af7beb911e'), ('DOI', '59a2f67b-201f-4ad3-83d1-2895315f91a1')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '59a2f67b-201f-4ad3-83d1-2895315f91a1', 'tail': 1993, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI'), ('EID', 'c3860dc4-c5a3-46ff-866d-764140797a30'), ('S2ID', '606eff1d-3d86-416e-9fe8-69af7beb911e'), ('DOI', '59a2f67b-201f-4ad3-83d1-2895315f91a1')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': '7006d559-83f0-4062-9851-112563d01126', 'weight': None, 'attributes': [('name', 'Lab of Pink Shrimp Three Deer')]}, {'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'One Sixteen', 'weight': None, 'attributes': None}, {'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =2, tails_len=2\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '59a2f67b-201f-4ad3-83d1-2895315f91a1', 'tail': 'Artificial Intelligence & Data Science Conference;Journal of Reinforcement Learning Strategies', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI'), ('EID', 'c3860dc4-c5a3-46ff-866d-764140797a30'), ('S2ID', '606eff1d-3d86-416e-9fe8-69af7beb911e'), ('DOI', '59a2f67b-201f-4ad3-83d1-2895315f91a1')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'weight': None, 'attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Computational Intelligence Journal', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'weight': None, 'attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Computational Intelligence Journal', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'tail': 'Computational Intelligence Journal', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '739f614a-2017-42ed-ba3e-ba42389f1168', 'weight': None, 'attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI How Transformers are Revolutionizing NLP Exploring Deep Learning for Autonomous Systems'), ('EID', 'd90ea739-5380-49e3-b4e0-b4569f43f95f'), ('S2ID', '30034bf8-80e4-47cf-9c56-cc6facd1694a'), ('DOI', '739f614a-2017-42ed-ba3e-ba42389f1168')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 2003, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Journal of Deep Learning Innovations;Global AI and Big Data Innovations', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '739f614a-2017-42ed-ba3e-ba42389f1168', 'weight': None, 'attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI How Transformers are Revolutionizing NLP Exploring Deep Learning for Autonomous Systems'), ('EID', 'd90ea739-5380-49e3-b4e0-b4569f43f95f'), ('S2ID', '30034bf8-80e4-47cf-9c56-cc6facd1694a'), ('DOI', '739f614a-2017-42ed-ba3e-ba42389f1168')]}, 'Year': {'entity': 2003, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Journal of Deep Learning Innovations;Global AI and Big Data Innovations', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '739f614a-2017-42ed-ba3e-ba42389f1168', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI How Transformers are Revolutionizing NLP Exploring Deep Learning for Autonomous Systems'), ('EID', 'd90ea739-5380-49e3-b4e0-b4569f43f95f'), ('S2ID', '30034bf8-80e4-47cf-9c56-cc6facd1694a'), ('DOI', '739f614a-2017-42ed-ba3e-ba42389f1168')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '739f614a-2017-42ed-ba3e-ba42389f1168', 'tail': 2003, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI How Transformers are Revolutionizing NLP Exploring Deep Learning for Autonomous Systems'), ('EID', 'd90ea739-5380-49e3-b4e0-b4569f43f95f'), ('S2ID', '30034bf8-80e4-47cf-9c56-cc6facd1694a'), ('DOI', '739f614a-2017-42ed-ba3e-ba42389f1168')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'c55a97d8-cf05-4014-b4d6-01730a79c2b4', 'weight': None, 'attributes': [('name', 'Laboratory of Apple Two Cedar')]}]\n", + "[{'entity': 'Four Birch', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '739f614a-2017-42ed-ba3e-ba42389f1168', 'tail': 'Journal of Deep Learning Innovations;Global AI and Big Data Innovations', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Self-Supervised Learning: The Next Big Thing in AI How Transformers are Revolutionizing NLP Exploring Deep Learning for Autonomous Systems'), ('EID', 'd90ea739-5380-49e3-b4e0-b4569f43f95f'), ('S2ID', '30034bf8-80e4-47cf-9c56-cc6facd1694a'), ('DOI', '739f614a-2017-42ed-ba3e-ba42389f1168')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': '6512799b-99f5-4967-bd6f-27b362ecc8e9', 'weight': None, 'attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities'), ('EID', '081f8efb-d812-4b7e-93e9-2ce40e46528a'), ('S2ID', '5c8bf977-37b5-46b5-b430-76967d41ee41'), ('DOI', '6512799b-99f5-4967-bd6f-27b362ecc8e9')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1996, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Journal of Deep Learning Innovations', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': '6512799b-99f5-4967-bd6f-27b362ecc8e9', 'weight': None, 'attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities'), ('EID', '081f8efb-d812-4b7e-93e9-2ce40e46528a'), ('S2ID', '5c8bf977-37b5-46b5-b430-76967d41ee41'), ('DOI', '6512799b-99f5-4967-bd6f-27b362ecc8e9')]}, 'Year': {'entity': 1996, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Journal of Deep Learning Innovations', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6512799b-99f5-4967-bd6f-27b362ecc8e9', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities'), ('EID', '081f8efb-d812-4b7e-93e9-2ce40e46528a'), ('S2ID', '5c8bf977-37b5-46b5-b430-76967d41ee41'), ('DOI', '6512799b-99f5-4967-bd6f-27b362ecc8e9')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6512799b-99f5-4967-bd6f-27b362ecc8e9', 'tail': 1996, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities'), ('EID', '081f8efb-d812-4b7e-93e9-2ce40e46528a'), ('S2ID', '5c8bf977-37b5-46b5-b430-76967d41ee41'), ('DOI', '6512799b-99f5-4967-bd6f-27b362ecc8e9')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'fcb9d5e2-a57f-45fa-b650-4a1a1e3d11fd', 'weight': None, 'attributes': [('name', 'Organization of Wolf Pasta Blue Oak')]}, {'entity': 'f05e4224-11eb-4c77-b9be-917a55eeb9b1', 'weight': None, 'attributes': [('name', 'University of Pink Four')]}]\n", + "[{'entity': 'Maroon Eight', 'weight': None, 'attributes': None}, {'entity': 'One Leopard', 'weight': None, 'attributes': None}]\n", + "heads_len =2, tails_len=2\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': '6512799b-99f5-4967-bd6f-27b362ecc8e9', 'tail': 'Journal of Deep Learning Innovations', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'The Intersection of AI and IoT in Smart Cities'), ('EID', '081f8efb-d812-4b7e-93e9-2ce40e46528a'), ('S2ID', '5c8bf977-37b5-46b5-b430-76967d41ee41'), ('DOI', '6512799b-99f5-4967-bd6f-27b362ecc8e9')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4', 'weight': None, 'attributes': [('Title', 'Graph Neural Networks for Fraud Detection Bayesian Optimization in Hyperparameter Tuning'), ('EID', '95ae76de-33af-4bb2-8352-0f6162e02ff6'), ('S2ID', '6a74ba9f-1d24-4737-9761-bca7fefbb445'), ('DOI', 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Journal of Intelligent Systems and Automation', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4', 'weight': None, 'attributes': [('Title', 'Graph Neural Networks for Fraud Detection Bayesian Optimization in Hyperparameter Tuning'), ('EID', '95ae76de-33af-4bb2-8352-0f6162e02ff6'), ('S2ID', '6a74ba9f-1d24-4737-9761-bca7fefbb445'), ('DOI', 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Journal of Intelligent Systems and Automation', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Graph Neural Networks for Fraud Detection Bayesian Optimization in Hyperparameter Tuning'), ('EID', '95ae76de-33af-4bb2-8352-0f6162e02ff6'), ('S2ID', '6a74ba9f-1d24-4737-9761-bca7fefbb445'), ('DOI', 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Graph Neural Networks for Fraud Detection Bayesian Optimization in Hyperparameter Tuning'), ('EID', '95ae76de-33af-4bb2-8352-0f6162e02ff6'), ('S2ID', '6a74ba9f-1d24-4737-9761-bca7fefbb445'), ('DOI', 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4', 'tail': 'Journal of Intelligent Systems and Automation', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Graph Neural Networks for Fraud Detection Bayesian Optimization in Hyperparameter Tuning'), ('EID', '95ae76de-33af-4bb2-8352-0f6162e02ff6'), ('S2ID', '6a74ba9f-1d24-4737-9761-bca7fefbb445'), ('DOI', 'f4ae4983-3b05-4fcd-aa30-c13f6c2510d4')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n", + "{'entity_type': 'Topic_ID', 'unique': True, 'from_column': 'Graph_Name'}\n", + "entity_map[FROM_COL] =Graph_Name\n", + "entity={'entity': 'Root_8_0', 'weight': None, 'attributes': None} for entity_map[ET] =Topic_ID\n", + "{'entity_type': 'Document_ID', 'from_column': 'doi', 'attribute_columns': [{'from_column': 'title', 'attribute_name': 'Title'}, {'from_column': 'eid', 'attribute_name': 'EID'}, {'from_column': 's2id', 'attribute_name': 'S2ID'}, {'from_column': 'doi', 'attribute_name': 'DOI'}], 'unique': True}\n", + "entity_map[FROM_COL] =doi\n", + "entity={'entity': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'weight': None, 'attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')]} for entity_map[ET] =Document_ID\n", + "{'entity_type': 'Year', 'from_column': 'year', 'attribute_columns': None, 'attribute_function': None, 'unique': True}\n", + "entity_map[FROM_COL] =year\n", + "entity={'entity': 1997, 'weight': None, 'attributes': None} for entity_map[ET] =Year\n", + "{'entity_type': 'Author_ID', 'from_column': 'author_ids', 'attribute_columns': [{'from_column': 'authors', 'attribute_name': 'Author_Name', 'retrival_operation': , 'args': None}], 'attribute_function': , 'args': None, 'unique': True}\n", + "entity_map[FROM_COL] =author_ids\n", + "entity={'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None} for entity_map[ET] =Author_ID\n", + "{'entity_type': 'Publisher', 'from_column': 'publication_name', 'unique': True}\n", + "entity_map[FROM_COL] =publication_name\n", + "entity={'entity': 'Computational Intelligence Journal', 'weight': None, 'attributes': None} for entity_map[ET] =Publisher\n", + "row_entities={'Topic_ID': {'entity': 'Root_8_0', 'weight': None, 'attributes': None}, 'Document_ID': {'entity': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'weight': None, 'attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')]}, 'Year': {'entity': 1997, 'weight': None, 'attributes': None}, 'Author_ID': {'entity': '0506d5ab-b679-415f-a3ce-40c762a73251', 'weight': None, 'attributes': None}, 'Publisher': {'entity': 'Computational Intelligence Journal', 'weight': None, 'attributes': None}}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'tail': 'Root_8_0', 'head_type': 'Document_ID', 'tail_type': 'Topic_ID', 'head_attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')], 'tail_attributes': None, 'weight': None, 'relation': 'part_of_topic'}\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'tail': 1997, 'head_type': 'Document_ID', 'tail_type': 'Year', 'head_attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')], 'tail_attributes': None, 'weight': None, 'relation': 'written_in_year'}\n", + "[{'entity': 'd1361bd1-ee0f-4550-99f9-4b6939fccfcd', 'weight': None, 'attributes': [('name', 'Laboratory of Wolf Beef')]}]\n", + "[{'entity': 'Sycamore Oak', 'weight': None, 'attributes': None}]\n", + "heads_len =1, tails_len=1\n", + "triple_details[HT] = Document_ID\n", + "triple_details = {'head': 'ce9218c4-c856-4aaf-b9cb-209234032600', 'tail': 'Computational Intelligence Journal', 'head_type': 'Document_ID', 'tail_type': 'Publisher', 'head_attributes': [('Title', 'Synthetic Data Generation for Training AI Models Cybersecurity Threat Detection Using AI Dimensionality Reduction Techniques for Big Data'), ('EID', 'c449f228-8421-483c-a32c-6bd6e09f75ba'), ('S2ID', '87404912-9946-4240-8f66-f36a679b08fb'), ('DOI', 'ce9218c4-c856-4aaf-b9cb-209234032600')], 'tail_attributes': None, 'weight': None, 'relation': 'published_by'}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 334/334 [00:00<00:00, 1101.31it/s]\n", + " 0%| | 0/9 [00:00 \u001b[39m\u001b[32m1\u001b[39m bundle = manager()\n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/block_manager.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 147\u001b[39m self.bundle = block(self.bundle)\n\u001b[32m 148\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m Exception:\n\u001b[32m 149\u001b[39m print(\u001b[33mf\"⚠️ Exception in block {block.tag}:\"\u001b[39m)\n\u001b[32m 150\u001b[39m traceback.print_exc()\n\u001b[32m--> \u001b[39m\u001b[32m151\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[32m 152\u001b[39m \n\u001b[32m 153\u001b[39m \u001b[38;5;66;03m# 4) Alias outputs: also store under the base tag (no NN_ prefix)\u001b[39;00m\n\u001b[32m 154\u001b[39m disp_tag = block.tag \u001b[38;5;66;03m# e.g., \"06_SemanticHNMFk\"\u001b[39;00m\n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/blocks/base_block.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(self, bundle)\u001b[39m\n\u001b[32m 162\u001b[39m \u001b[38;5;66;03m# 2) run block\u001b[39;00m\n\u001b[32m 163\u001b[39m \u001b[38;5;66;03m# --------------------------------------------------------------\u001b[39;00m\n\u001b[32m 164\u001b[39m self._pending_ckpt_map: Dict[str, str] = {}\n\u001b[32m 165\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m self._io_path_rewriter():\n\u001b[32m--> \u001b[39m\u001b[32m166\u001b[39m self.run(bundle)\n\u001b[32m 167\u001b[39m \n\u001b[32m 168\u001b[39m \u001b[38;5;66;03m# --------------------------------------------------------------\u001b[39;00m\n\u001b[32m 169\u001b[39m \u001b[38;5;66;03m# 3) verify outputs\u001b[39;00m\n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/blocks/termite_neo4j_block.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(self, bundle)\u001b[39m\n\u001b[32m 398\u001b[39m termite.from_csv_to_triplets(str(data_csv_path), str(data_triplets_path), data_triplet_map)\n\u001b[32m 399\u001b[39m termite.update_database_multithreaded(str(data_triplets_path))\n\u001b[32m 400\u001b[39m \n\u001b[32m 401\u001b[39m \u001b[38;5;66;03m# ---------- PASS 2: TOPIC triplets ----------\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m402\u001b[39m termite.from_csv_to_triplets(str(topic_csv_path), str(topic_triplets_path), topic_triplet_map)\n\u001b[32m 403\u001b[39m termite.update_database_multithreaded(str(topic_triplets_path))\n\u001b[32m 404\u001b[39m \n\u001b[32m 405\u001b[39m \u001b[38;5;66;03m# ---------- Register outputs ----------\u001b[39;00m\n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/applications/Termite/termite.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(self, csv_path, save_path, column_triplet_map)\u001b[39m\n\u001b[32m 116\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m from_csv_to_triplets(self, csv_path: str, save_path: str, column_triplet_map: Optional[Mapping] = \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m 117\u001b[39m \"\"\"\n\u001b[32m 118\u001b[39m Builds a datafile that maps the raw csv into a head-relation-tail csv\n\u001b[32m 119\u001b[39m \"\"\"\n\u001b[32m--> \u001b[39m\u001b[32m120\u001b[39m self.graph_injector.from_csv_to_triplets(csv_path, save_path, column_triplet_map)\n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/applications/Termite/neo4j_termite/DataInjector.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(self, csv_path, save_path, column_triplet_map)\u001b[39m\n\u001b[32m 309\u001b[39m self.add_triple(docs, **triple_details)\n\u001b[32m 310\u001b[39m \n\u001b[32m 311\u001b[39m \u001b[38;5;66;03m# The tails only are embedded inline of the data row, and must be exctracted through the function passed\u001b[39;00m\n\u001b[32m 312\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m tail_extraction:\n\u001b[32m--> \u001b[39m\u001b[32m313\u001b[39m tail_entities = tail_extraction(tail_args ) \u001b[38;5;66;03m# extraction call, must return a list of RETURN_TYPE\u001b[39;00m\n\u001b[32m 314\u001b[39m triple_details[H] = row_entities[triple_details[HT]][ENTITY] \u001b[38;5;66;03m# Head is the Row entity looked up through the head type in the details\u001b[39;00m\n\u001b[32m 315\u001b[39m triple_details[HA] = row_entities[triple_details[HT]][ATTRIBUTES]\n\u001b[32m 316\u001b[39m \n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/TELF/pipeline/blocks/termite_neo4j_block.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(args)\u001b[39m\n\u001b[32m 67\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m get_parent_topic(args):\n\u001b[32m 68\u001b[39m data_string = args[\u001b[33m'data'\u001b[39m]\n\u001b[32m 69\u001b[39m returnable_entities = []\n\u001b[32m---> \u001b[39m\u001b[32m70\u001b[39m parent_name = data_string.parent_name\n\u001b[32m 71\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m type(parent_name) == str:\n\u001b[32m 72\u001b[39m entity_returnable = deepcopy(RETURN_TYPE)\n\u001b[32m 73\u001b[39m entity_returnable[ENTITY] = parent_name\n", + "\u001b[32m~/anaconda3/envs/TELF/lib/python3.11/site-packages/pandas/core/generic.py\u001b[39m in \u001b[36m?\u001b[39m\u001b[34m(self, name)\u001b[39m\n\u001b[32m 6314\u001b[39m \u001b[38;5;28;01mand\u001b[39;00m name \u001b[38;5;28;01mnot\u001b[39;00m \u001b[38;5;28;01min\u001b[39;00m self._accessors\n\u001b[32m 6315\u001b[39m \u001b[38;5;28;01mand\u001b[39;00m self._info_axis._can_hold_identifiers_and_holds_name(name)\n\u001b[32m 6316\u001b[39m ):\n\u001b[32m 6317\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m self[name]\n\u001b[32m-> \u001b[39m\u001b[32m6318\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m object.__getattribute__(self, name)\n", + "\u001b[31mAttributeError\u001b[39m: 'Series' object has no attribute 'parent_name'" + ] + } + ], + "source": [ + "bundle = manager()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ba02ebb", + "metadata": {}, + "outputs": [], + "source": [ + "bundle.keys()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "TELF", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/post_install/__init__.py b/post_install/__init__.py new file mode 100644 index 00000000..9fd61f5e --- /dev/null +++ b/post_install/__init__.py @@ -0,0 +1 @@ +from .__main__ import main, install_chrome_cli \ No newline at end of file diff --git a/post_install/__main__.py b/post_install/__main__.py new file mode 100644 index 00000000..06b1b6c2 --- /dev/null +++ b/post_install/__main__.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +""" +Post-install helper for TELF. + +- Installs (or verifies) spaCy + models and NLTK data in the *current* Python env. +- Optional GPU/HPC helpers via flags (uses conda consistently with -y). +- Optional Kaleido/Chrome setup for Plotly static image export. +- Avoids changing NumPy versions or bypassing your resolver unless requested. + +Usage examples: + python post_install.py # NLP models + NLTK (default) + python post_install.py --kaleido-chrome # also prepare Chrome for Kaleido + python post_install.py --gpu # add CuPy via conda-forge + python post_install.py --hpc-conda # add mpi4py via conda-forge + python post_install.py --skip-models # skip spaCy/NLTK bits + +As a console script (when exposed via pyproject): + telf-post-install --kaleido-chrome +""" + +import argparse +import importlib.util +import os +import subprocess +import sys +from importlib.metadata import PackageNotFoundError, version +from shutil import which + +# ------------------------------ +# Utilities +# ------------------------------ + +def run(cmd, **kw): + """Run a command with check=True and echo it.""" + if isinstance(cmd, (list, tuple)): + printable = " ".join(str(x) for x in cmd) + else: + printable = str(cmd) + print(">", printable) + subprocess.run(cmd, check=True, **kw) + + +def has_module(mod_name: str) -> bool: + return importlib.util.find_spec(mod_name) is not None + + +def ensure_pkg(import_name: str, pip_name: str | None = None, version_str: str | None = None): + """ + Ensure `import_name` can be imported. If not, pip-install into THIS interpreter. + """ + try: + __import__(import_name) + except ModuleNotFoundError: + to_install = pip_name or import_name + if version_str: + to_install = f"{to_install}=={version_str}" + run([sys.executable, "-m", "pip", "install", to_install]) + + +# ------------------------------ +# NLP bits (spaCy + NLTK) +# ------------------------------ + +def ensure_spacy_and_models(): + """ + Make sure spaCy is present and large/transformer models are available. + Model downloads are skipped if already installed. + """ + # Keep versions aligned with pyproject (adjust if you bump there) + ensure_pkg("spacy", "spacy", "3.8.2") + + # Only install nltk if we’re actually going to download NLTK data later. + ensure_pkg("nltk", "nltk", "3.9.1") + + # Install spaCy models only if missing + if not has_module("en_core_web_lg"): + run([sys.executable, "-m", "spacy", "download", "en_core_web_lg"]) + else: + print("spaCy model en_core_web_lg already present; skipping.") + + if not has_module("en_core_web_trf"): + run([sys.executable, "-m", "spacy", "download", "en_core_web_trf"]) + else: + print("spaCy model en_core_web_trf already present; skipping.") + + +def download_nltk_data(): + """ + Download NLTK corpora via API (more robust than `-m nltk.downloader`). + Skips re-downloads. + """ + import nltk + + for pkg in ("wordnet", "omw-1.4"): + print(f"Ensuring NLTK data: {pkg}") + nltk.download(pkg, quiet=True) + + +# ------------------------------ +# GPU / HPC optional helpers +# ------------------------------ + +def conda_required(): + if which("conda") is None: + raise RuntimeError( + "Conda is required for the requested GPU/HPC (conda) operations, " + "but 'conda' was not found on PATH." + ) + + +def conda_install(*packages: str, channel: str = "conda-forge"): + """ + Install conda packages non-interactively from a consistent channel. + """ + conda_required() + run(["conda", "install", "-y", "-c", channel, *packages]) + + +def install_gpu_dependencies(via_conda_toolkit: bool, install_cupy: bool): + """ + Optionally install CUDA toolkit pieces and CuPy. Uses conda-forge consistently. + """ + if via_conda_toolkit: + print("Installing CUDA toolkit components (conda-forge)...") + conda_install("cudatoolkit", channel="conda-forge") + conda_install("cudnn", channel="conda-forge") + + if install_cupy: + print("Installing CuPy (conda-forge)...") + conda_install("cupy", channel="conda-forge") + + +def install_mpi(hpc_pip: bool, hpc_conda: bool): + """ + Optionally install mpi4py either via pip (requires system MPI toolchain) + or via conda-forge (preferred for portability). + """ + if hpc_pip and hpc_conda: + # Prefer conda-forge to avoid system MPI mismatches + print("Both --hpc and --hpc-conda were set; preferring conda-forge build.") + hpc_pip = False + + if hpc_conda: + print("Installing mpi4py via conda-forge...") + conda_install("mpi4py", channel="conda-forge") + elif hpc_pip: + print("Installing mpi4py via pip (requires system MPI headers/libs)...") + run([sys.executable, "-m", "pip", "install", "mpi4py"]) + + +# ------------------------------ +# Kaleido / Chrome for Plotly static export +# ------------------------------ + +MIN_PLOTLY = "6.1.1" # Kaleido v1 requires Plotly >= 6.1.1 +MIN_KALEIDO = "1.1.0" # Provides get_chrome_sync() + +def _ensure_packaging(): + try: + import packaging # noqa: F401 + except ModuleNotFoundError: + run([sys.executable, "-m", "pip", "install", "packaging"]) + + +def _pkg_version_ok(dist: str, min_ver: str) -> bool: + _ensure_packaging() + from packaging.version import Version + try: + return Version(version(dist)) >= Version(min_ver) + except PackageNotFoundError: + return False + + +def ensure_plotly_kaleido_versions(): + """ + Ensure plotly/kaleido are installed and new enough for Kaleido v1. + """ + if not _pkg_version_ok("plotly", MIN_PLOTLY): + run([sys.executable, "-m", "pip", "install", f"plotly>={MIN_PLOTLY}"]) + if not _pkg_version_ok("kaleido", MIN_KALEIDO): + run([sys.executable, "-m", "pip", "install", f"kaleido>={MIN_KALEIDO}"]) + + +def _try_cli_chrome_fetch(): + """ + Fallback to CLI helpers if available: plotly_get_chrome / kaleido_get_chrome. + """ + for cli in ("plotly_get_chrome", "kaleido_get_chrome"): + path = which(cli) + if path: + try: + run([path]) + return True + except subprocess.CalledProcessError: + pass + return False + + +def ensure_chrome_for_kaleido(): + """ + Ensure Chrome is available for Kaleido v1+. + + Behavior: + - If BROWSER_PATH points to a real file, use it. + - Else, try kaleido.get_chrome_sync() to download a portable Chrome. + - Else, try CLI helpers (plotly_get_chrome / kaleido_get_chrome). + - Raises on failure. + """ + ensure_plotly_kaleido_versions() + + bp = os.environ.get("BROWSER_PATH") + if bp and os.path.exists(bp): + print(f"BROWSER_PATH already set: {bp}") + return + + # Preferred: Python helper returns a path; we also set BROWSER_PATH. + try: + import kaleido + if hasattr(kaleido, "get_chrome_sync"): + print("Preparing Chrome for Kaleido (this may download a portable binary)...") + path = kaleido.get_chrome_sync() + os.environ["BROWSER_PATH"] = str(path) + print(f"✅ Chrome ready for Kaleido at: {path}") + return + except ModuleNotFoundError: + # Shouldn't happen; ensure_plotly_kaleido_versions installed it. + run([sys.executable, "-m", "pip", "install", f"kaleido>={MIN_KALEIDO}"]) + import kaleido # noqa: F401 + + # Fallback: try CLI helpers + if _try_cli_chrome_fetch(): + print("Chrome prepared via CLI helper.") + return + + raise RuntimeError( + "Could not prepare Chrome for Kaleido. " + "Set BROWSER_PATH to your Chrome/Chromium binary, or run plotly_get_chrome." + ) + + +def install_chrome_cli(): + """ + Small entrypoint for a dedicated console script: + poetry run telf-install-chrome + """ + ensure_chrome_for_kaleido() + + +# ------------------------------ +# Orchestration +# ------------------------------ + +def run_post_install_commands( + gpu: bool = False, + hpc: bool = False, + hpc_conda: bool = False, + gpu_toolkit: bool = False, + skip_models: bool = False, + kaleido_chrome: bool = False, +): + """ + Execute post-install steps in a safe, idempotent way. + """ + # 0) Kaleido/Chrome (optional but recommended if you need PNG/PDF export) + if kaleido_chrome: + ensure_chrome_for_kaleido() + + # 1) NLP bits (spaCy + models, NLTK data) + if not skip_models: + ensure_spacy_and_models() + download_nltk_data() + else: + print("Skipping spaCy model and NLTK data steps (--skip-models).") + + # 2) GPU deps (optional) + if gpu_toolkit or gpu: + install_gpu_dependencies(via_conda_toolkit=gpu_toolkit, install_cupy=gpu) + + # 3) HPC MPI (optional) + install_mpi(hpc_pip=hpc, hpc_conda=hpc_conda) + + print("Post-install completed successfully.") + + +def main(): + p = argparse.ArgumentParser( + description="Post installation script for TELF (models/data, optional GPU/HPC extras, and Kaleido/Chrome setup)." + ) + p.add_argument("--gpu", action="store_true", help="Install CuPy via conda-forge.") + p.add_argument( + "--gpu-toolkit", + action="store_true", + help="Install cudatoolkit and cudnn via conda-forge.", + ) + p.add_argument( + "--hpc", + action="store_true", + help="Install mpi4py via pip (requires compatible system MPI).", + ) + p.add_argument( + "--hpc-conda", + action="store_true", + help="Install mpi4py via conda-forge (preferred for portability).", + ) + p.add_argument( + "--skip-models", + action="store_true", + help="Skip spaCy model downloads and NLTK data steps.", + ) + p.add_argument( + "--kaleido-chrome", + action="store_true", + help="Ensure Chrome is available for plotly+kaleido static image export.", + ) + + args = p.parse_args() + try: + run_post_install_commands( + gpu=args.gpu, + hpc=args.hpc, + hpc_conda=args.hpc_conda, + gpu_toolkit=args.gpu_toolkit, + skip_models=args.skip_models, + kaleido_chrome=args.kaleido_chrome, + ) + except subprocess.CalledProcessError as e: + print(f"\nCommand failed with exit code {e.returncode}:\n {' '.join(e.cmd)}") + sys.exit(e.returncode) + except RuntimeError as e: + print(f"\nERROR: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 5b91930d..0572a3e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,8 @@ readme = "README.md" # Explicitly tell Poetry where to find the Python package packages = [ - { include = "TELF" } + { include = "TELF" }, + { include = "post_install" } ] [tool.poetry.dependencies] @@ -33,7 +34,8 @@ spacy = "^3.8.2" mat73 = "^0.65" spacy-transformers = "^1.3.5" rapidfuzz = "^3.10.0" -plotly = "^6.0.0" +plotly = ">=6.1.1" +kaleido = "^1.1.0" wordcloud = "^1.9.4" httpx = "^0.28.1" xmltodict = "^0.14.2" @@ -60,7 +62,12 @@ jupyterlab = "^4.2.5" notebook = "^7.2.2" ipywidgets = "^8.1.5" opensearch-py = "^3.0.0" + [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +telf-post-install = "post_install:main" +telf-install-chrome = "post_install:install_chrome_cli" \ No newline at end of file From 59484850a37b991eeb0edd274d9e30eab9862015 Mon Sep 17 00:00:00 2001 From: Ryan Calvin Barron Date: Wed, 24 Sep 2025 17:36:11 -0600 Subject: [PATCH 03/13] post install script move --- .gitignore | 4 +- post_install.py | 201 ------------------------------------------------ 2 files changed, 3 insertions(+), 202 deletions(-) delete mode 100644 post_install.py diff --git a/.gitignore b/.gitignore index bbec2b31..540fce89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +projects example_out example_results hidden_keys.py @@ -15,7 +16,8 @@ EXAMPLE_OUT result_example/ search_terms.md scopus_cache/ - +examples/Termite/01_termite_output +examples/Termite/DOCKERDATA/ # mac results/ poetry.lock diff --git a/post_install.py b/post_install.py deleted file mode 100644 index d6bf1936..00000000 --- a/post_install.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -""" -Post-install helper for TELF. - -- Installs (or verifies) spaCy + models and NLTK data in the *current* Python env. -- Optional GPU/HPC helpers via flags (uses conda consistently with -y). -- Avoids changing NumPy versions or bypassing your resolver. - -Usage examples: - python post_install.py - python post_install.py --gpu - python post_install.py --hpc-conda - python post_install.py --gpu --gpu-toolkit -""" - -import argparse -import importlib.util -import subprocess -import sys -from shutil import which - - -def run(cmd, **kw): - """Run a command with check=True and echo it.""" - print(">", " ".join(cmd)) - subprocess.run(cmd, check=True, **kw) - - -def has_module(mod_name: str) -> bool: - return importlib.util.find_spec(mod_name) is not None - - -def ensure_pkg(import_name: str, pip_name: str | None = None, version: str | None = None): - """ - Ensure `import_name` can be imported. If not, pip-install into THIS interpreter. - """ - try: - __import__(import_name) - except ModuleNotFoundError: - to_install = pip_name or import_name - if version: - to_install = f"{to_install}=={version}" - run([sys.executable, "-m", "pip", "install", to_install]) - - -def ensure_spacy_and_models(): - """ - Make sure spaCy is present and large/transformer models are available. - Model downloads are skipped if already installed. - """ - # Keep versions aligned with your pyproject (adjust if you bump there) - ensure_pkg("spacy", "spacy", "3.8.2") - - # Only install nltk if we’re actually going to download NLTK data later. - # We do it here so the import is guaranteed to work for the downloader. - ensure_pkg("nltk", "nltk", "3.9.1") - - # Install spaCy models only if missing - if not has_module("en_core_web_lg"): - run([sys.executable, "-m", "spacy", "download", "en_core_web_lg"]) - else: - print("spaCy model en_core_web_lg already present; skipping.") - - if not has_module("en_core_web_trf"): - run([sys.executable, "-m", "spacy", "download", "en_core_web_trf"]) - else: - print("spaCy model en_core_web_trf already present; skipping.") - - -def download_nltk_data(): - """ - Download NLTK corpora via API (more robust than `-m nltk.downloader`). - Skips re-downloads. - """ - import nltk - - for pkg in ("wordnet", "omw-1.4"): - print(f"Ensuring NLTK data: {pkg}") - nltk.download(pkg, quiet=True) - - -def conda_required(): - if which("conda") is None: - raise RuntimeError( - "Conda is required for the requested GPU/HPC (conda) operations, " - "but 'conda' was not found on PATH." - ) - - -def conda_install(*packages: str, channel: str = "conda-forge"): - """ - Install conda packages non-interactively from a consistent channel. - """ - conda_required() - run(["conda", "install", "-y", "-c", channel, *packages]) - - -def install_gpu_dependencies(via_conda_toolkit: bool, install_cupy: bool): - """ - Optionally install CUDA toolkit pieces and CuPy. Uses conda-forge consistently. - """ - if via_conda_toolkit: - print("Installing CUDA toolkit components (conda-forge)...") - conda_install("cudatoolkit", channel="conda-forge") - conda_install("cudnn", channel="conda-forge") - - if install_cupy: - print("Installing CuPy (conda-forge)...") - conda_install("cupy", channel="conda-forge") - - -def install_mpi(hpc_pip: bool, hpc_conda: bool): - """ - Optionally install mpi4py either via pip (requires system MPI toolchain) - or via conda-forge (preferred for portability). - """ - if hpc_pip and hpc_conda: - # Prefer conda-forge to avoid system MPI mismatches - print("Both --hpc and --hpc-conda were set; preferring conda-forge build.") - hpc_pip = False - - if hpc_conda: - print("Installing mpi4py via conda-forge...") - conda_install("mpi4py", channel="conda-forge") - elif hpc_pip: - print("Installing mpi4py via pip (requires system MPI headers/libs)...") - run([sys.executable, "-m", "pip", "install", "mpi4py"]) - - -def run_post_install_commands( - gpu: bool = False, - hpc: bool = False, - hpc_conda: bool = False, - gpu_toolkit: bool = False, - skip_models: bool = False, -): - """ - Execute post-install steps in a safe, idempotent way. - """ - # 1) NLP bits (spaCy + models, NLTK data) - if not skip_models: - ensure_spacy_and_models() - download_nltk_data() - else: - print("Skipping spaCy model and NLTK data steps (--skip-models).") - - # 2) GPU deps (optional) - if gpu_toolkit or gpu: - install_gpu_dependencies(via_conda_toolkit=gpu_toolkit, install_cupy=gpu) - - # 3) HPC MPI (optional) - install_mpi(hpc_pip=hpc, hpc_conda=hpc_conda) - - print("Post-install completed successfully.") - - -def main(): - p = argparse.ArgumentParser( - description="Post installation script for downloading models/data and optional GPU/HPC extras." - ) - p.add_argument("--gpu", action="store_true", help="Install CuPy via conda-forge.") - p.add_argument( - "--gpu-toolkit", - action="store_true", - help="Install cudatoolkit and cudnn via conda-forge.", - ) - p.add_argument( - "--hpc", - action="store_true", - help="Install mpi4py via pip (requires compatible system MPI).", - ) - p.add_argument( - "--hpc-conda", - action="store_true", - help="Install mpi4py via conda-forge (preferred for portability).", - ) - p.add_argument( - "--skip-models", - action="store_true", - help="Skip spaCy model downloads and NLTK data steps.", - ) - - args = p.parse_args() - try: - run_post_install_commands( - gpu=args.gpu, - hpc=args.hpc, - hpc_conda=args.hpc_conda, - gpu_toolkit=args.gpu_toolkit, - skip_models=args.skip_models, - ) - except subprocess.CalledProcessError as e: - print(f"\nCommand failed with exit code {e.returncode}:\n {' '.join(e.cmd)}") - sys.exit(e.returncode) - except RuntimeError as e: - print(f"\nERROR: {e}") - sys.exit(1) - - -if __name__ == "__main__": - main() From b8dd18d5c0c60715a8024c6dae55db5d98bd4ee0 Mon Sep 17 00:00:00 2001 From: Ryan Calvin Barron Date: Thu, 25 Sep 2025 16:02:13 -0600 Subject: [PATCH 04/13] NER entittes to termite KG injection --- TELF/pipeline/__init__.py | 4 +- TELF/pipeline/blocks/__init__.py | 1 + TELF/pipeline/blocks/artic_fox_block.py | 2 +- .../blocks/author_affiliation_tables.py | 141 ++ .../pipeline/blocks/block_helpers/__init__.py | 0 .../block_helpers/affiliation_partition.py | 179 +++ .../blocks/block_helpers/author_partition.py | 276 ++++ .../blocks/block_helpers/hnmfk_paths.py | 51 + .../blocks/block_helpers/peacock_renderer.py | 267 ++++ .../blocks/collect_hnmfk_leaf_block.py | 8 +- TELF/pipeline/blocks/peacock_stats_block.py | 288 +--- TELF/pipeline/blocks/spacey_NER_block.py | 118 +- TELF/pipeline/blocks/termite_neo4j_block.py | 787 +++++++--- TELF/pipeline/blocks/termite_vector_block.py | 2 +- ...mantic_hnmfk_collection_slurm_option.ipynb | 1369 ++--------------- 15 files changed, 1726 insertions(+), 1767 deletions(-) create mode 100644 TELF/pipeline/blocks/author_affiliation_tables.py create mode 100644 TELF/pipeline/blocks/block_helpers/__init__.py create mode 100644 TELF/pipeline/blocks/block_helpers/affiliation_partition.py create mode 100644 TELF/pipeline/blocks/block_helpers/author_partition.py create mode 100644 TELF/pipeline/blocks/block_helpers/hnmfk_paths.py create mode 100644 TELF/pipeline/blocks/block_helpers/peacock_renderer.py diff --git a/TELF/pipeline/__init__.py b/TELF/pipeline/__init__.py index 1da1e48e..6cbd978a 100644 --- a/TELF/pipeline/__init__.py +++ b/TELF/pipeline/__init__.py @@ -70,4 +70,6 @@ from .blocks.collect_hnmfk_leaf_block import CollectHNMFkLeafBlock from .blocks.termite_neo4j_block import TermiteNeo4jBlock -from .blocks.termite_vector_block import TermiteVectorBlock \ No newline at end of file +from .blocks.termite_vector_block import TermiteVectorBlock + +from .blocks.author_affiliation_tables import AffiliationsAndAuthorsBlock diff --git a/TELF/pipeline/blocks/__init__.py b/TELF/pipeline/blocks/__init__.py index 69b73654..9b6b984b 100644 --- a/TELF/pipeline/blocks/__init__.py +++ b/TELF/pipeline/blocks/__init__.py @@ -63,3 +63,4 @@ from .collect_hnmfk_leaf_block import CollectHNMFkLeafBlock from .termite_neo4j_block import TermiteNeo4jBlock from .termite_vector_block import TermiteVectorBlock +from .author_affiliation_tables import AffiliationsAndAuthorsBlock \ No newline at end of file diff --git a/TELF/pipeline/blocks/artic_fox_block.py b/TELF/pipeline/blocks/artic_fox_block.py index 8e9fc59d..d882221a 100644 --- a/TELF/pipeline/blocks/artic_fox_block.py +++ b/TELF/pipeline/blocks/artic_fox_block.py @@ -68,7 +68,7 @@ def run(self, bundle: DataBundle) -> None: model.load_model() pipeline = ArcticFox(model=model, **self.init_settings) - # pipeline.run_full_pipeline(data_df=df, vocab=vocabulary, **self.call_settings) + pipeline.run_full_pipeline(data_df=df, vocab=vocabulary, **self.call_settings) status_value = "Done" diff --git a/TELF/pipeline/blocks/author_affiliation_tables.py b/TELF/pipeline/blocks/author_affiliation_tables.py new file mode 100644 index 00000000..6dce1f4d --- /dev/null +++ b/TELF/pipeline/blocks/author_affiliation_tables.py @@ -0,0 +1,141 @@ +# blocks/affiliations_and_authors_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Dict, Sequence, Any, Optional, List, Tuple, Union + +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +# your helpers live in blocks/block_helpers/ +from .block_helpers.affiliation_partition import generate_top_affiliations_with_country +from .block_helpers.author_partition import write_top_authors_by_cluster + + +class AffiliationsAndAuthorsBlock(AnimalBlock): + """ + Compute (affiliation, country, year) paper counts and top authors by cluster. + + ───────────────────────────────────────────────────────────── + always needs : ('df',) – accepts a CSV path OR a pandas.DataFrame + provides : ('affiliations_df', 'affiliations_csv', + 'authors_df', 'authors_csv') + tag : 'AffilsAndAuthors' (namespace for its outputs) + + Results are written under // . + Checkpoints persist the two CSV paths so the block can be skipped on re-run. + """ + + CANONICAL_NEEDS = ("df",) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("affiliations_df", "affiliations_csv", + "authors_df", "authors_csv"), + # Persist only the CSVs; DataFrames are rebuilt on load if needed. + checkpoint_keys: Sequence[str] = ("affiliations_csv", "authors_csv"), + conditional_needs: Sequence[tuple[str, Any]] = (), + tag: str = "AffilsAndAuthors", + # Defaults mirror your helper signatures + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw: Any, + ) -> None: + + default_init: Dict[str, Any] = {} + + default_call: Dict[str, Any] = { + # generate_top_affiliations_with_country(...) + "min_total_papers": 20, + "country_filter": None, # exact match (including 'unknown') or None + "partition_by_year": False, + "per_year_output_dir": None, # if None and partition_by_year=True → use /by_year + + # write_top_authors_by_cluster(...) + "countries": None, # list[str] or None + "top_n": 10, + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=list(conditional_needs or []), + checkpoint_keys=checkpoint_keys, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + # ───────────────────────────────────────────────────────────── + # helpers + # ───────────────────────────────────────────────────────────── + def _ensure_input_csv(self, bundle: DataBundle) -> Path: + """ + Accepts either a DataFrame or a path in bundle['df']. + If a DataFrame, persist it to //input.csv and return that path. + """ + src = bundle[self.needs[0]] + save_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + save_dir.mkdir(parents=True, exist_ok=True) + + if isinstance(src, pd.DataFrame): + inp = save_dir / "input.csv" + src.to_csv(inp, index=False, encoding="utf-8-sig") + return inp + + # let AnimalBlock’s path rewriter handle legacy numbered dirs + return Path(src) + + # ───────────────────────────────────────────────────────────── + # work + # ───────────────────────────────────────────────────────────── + def run(self, bundle: DataBundle) -> None: + df_path = self._ensure_input_csv(bundle) + + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + + # === 1) Affiliations with country (and per-year optional) === + affils_csv = out_dir / "affiliations_top.csv" + per_year_dir = self.call_settings.get("per_year_output_dir") + if self.call_settings.get("partition_by_year") and not per_year_dir: + per_year_dir = out_dir / "by_year" + + generate_top_affiliations_with_country( + df_path=df_path, + affils_output_path=affils_csv, + min_total_papers=int(self.call_settings["min_total_papers"]), + country_filter=self.call_settings.get("country_filter"), + partition_by_year=bool(self.call_settings.get("partition_by_year")), + per_year_output_dir=per_year_dir, + ) + + # read back for the bundle + aff_df = pd.read_csv(affils_csv) if affils_csv.is_file() else pd.DataFrame( + columns=["affiliation_name", "country", "year", "paper_count"] + ) + + # register checkpoints / provide + self.register_checkpoint("affiliations_csv", affils_csv) + bundle[f"{self.tag}.affiliations_csv"] = str(affils_csv) + bundle[f"{self.tag}.affiliations_df"] = aff_df + + # === 2) Top authors by cluster ============================= + authors_csv = out_dir / "top_authors_by_cluster.csv" + # Note: helper param name is COUNTY_NAMES (kept as-is) + result_df = write_top_authors_by_cluster( + df_path=str(df_path), + output_path=str(authors_csv), + COUNTY_NAMES=self.call_settings.get("countries"), + top_n=int(self.call_settings.get("top_n", 10)), + ) + + # register checkpoints / provide + self.register_checkpoint("authors_csv", authors_csv) + bundle[f"{self.tag}.authors_csv"] = str(authors_csv) + bundle[f"{self.tag}.authors_df"] = result_df \ No newline at end of file diff --git a/TELF/pipeline/blocks/block_helpers/__init__.py b/TELF/pipeline/blocks/block_helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TELF/pipeline/blocks/block_helpers/affiliation_partition.py b/TELF/pipeline/blocks/block_helpers/affiliation_partition.py new file mode 100644 index 00000000..8329ae2e --- /dev/null +++ b/TELF/pipeline/blocks/block_helpers/affiliation_partition.py @@ -0,0 +1,179 @@ +import ast +import json +from pathlib import Path +import pandas as pd + +UNKNOWN_COUNTRY = "unknown" + +def _parse_affiliations_with_country(raw) -> list[tuple[str, str]]: + """ + Parse a JSON/Python-literal dict into [(name, country), …]. + Always returns a country (missing ⇒ 'unknown'). Returns [] if unparseable. + Expected shape (examples): + '{"0": {"name": "MIT", "country": "United States"}}' + '{0: {"name": "LANL"}}' + """ + if raw is None: + return [] + try: + if pd.isna(raw): # safe even if raw isn't a pandas scalar + return [] + except Exception: + pass + + if isinstance(raw, dict): + parsed = raw + else: + s = str(raw).strip() + if s in ("", "{}", "[]"): + return [] + try: + parsed = json.loads(s) + except Exception: + try: + parsed = ast.literal_eval(s) + except Exception: + return [] + + if not isinstance(parsed, dict): + return [] + + out: list[tuple[str, str]] = [] + for info in parsed.values(): + if not isinstance(info, dict): + continue + name = info.get("name") + if not isinstance(name, str) or not name.strip(): + continue + country = info.get("country") + if isinstance(country, str): + country = country.strip() or UNKNOWN_COUNTRY + elif country is None: + country = UNKNOWN_COUNTRY + else: + country = str(country).strip() or UNKNOWN_COUNTRY + out.append((name.strip(), country)) + return out + +def generate_top_affiliations_with_country( + df_path: str | Path, + affils_output_path: str | Path, + min_total_papers: int = 20, + country_filter: str | None = None, # ← filter to exactly this country (or 'unknown') + partition_by_year: bool = False, # ← also emit one CSV per year + per_year_output_dir: str | Path | None = None, +): + """ + Reads df_path (must have 'year' and 'affiliations'), computes per-(affiliation, country, year) + paper counts for affiliations whose total_papers (within the current filter) ≥ min_total_papers. + Always includes a 'country' value; if missing in the source, uses 'unknown'. + + If country_filter is provided, restricts to that single country (exact string match, including 'unknown'). + """ + df = pd.read_csv(df_path) + + if 'year' not in df.columns: + raise KeyError("Expected a 'year' column.") + df['year'] = pd.to_numeric(df['year'], errors='coerce') + df = df[df['year'].notna()].copy() + df['year'] = df['year'].astype(int) + + if 'affiliations' not in df.columns: + raise KeyError("Expected an 'affiliations' column.") + + # 1) Parse affiliations → list of (name, country or 'unknown') + df_aff = df.copy() + df_aff['affil_tuples'] = df_aff['affiliations'].apply(_parse_affiliations_with_country) + + # 2) Explode into one row per (paper, affiliation_name, country) + exploded_aff = df_aff.explode('affil_tuples') + exploded_aff = exploded_aff[exploded_aff['affil_tuples'].notna()].copy() + exploded_aff[['affiliation_name', 'country']] = pd.DataFrame( + exploded_aff['affil_tuples'].tolist(), index=exploded_aff.index + ) + # Defensive fill (should already be set by parser) + exploded_aff['country'] = exploded_aff['country'].fillna(UNKNOWN_COUNTRY) + + # 3) Optional: restrict to one specific country + if country_filter is not None: + exploded_aff = exploded_aff[exploded_aff['country'] == country_filter].copy() + + # 4) Totals per affiliation (within current filter scope) + total_per_aff = ( + exploded_aff + .groupby(['affiliation_name', 'country']) + .size() + .reset_index(name='total_papers') + .sort_values('total_papers', ascending=False) + ) + + print("=== Affiliations", + f"in [{country_filter}]" if country_filter is not None else "(all countries)", + "with their total paper counts ===") + print(total_per_aff.head(20).to_string(index=False)) + print("───────────────────────────────────────────────────────────────────────────\n") + + # 5) Keep affiliations with ≥ min_total_papers + top_affils = total_per_aff[total_per_aff['total_papers'] >= min_total_papers][ + ['affiliation_name', 'country'] + ] + + if top_affils.empty: + print(f"No affiliation{' in ' + country_filter if country_filter else ''} " + f"meets ≥ {min_total_papers} total papers.") + aff_year_counts = pd.DataFrame(columns=['affiliation_name','country','year','paper_count']) + else: + # 6) Per-year counts for top affiliations + exploded_aff_top = exploded_aff.merge(top_affils, on=['affiliation_name','country'], how='inner') + aff_year_counts = ( + exploded_aff_top + .groupby(['affiliation_name','country','year']) + .size() + .reset_index(name='paper_count') + .sort_values(['affiliation_name','country','year']) + .reset_index(drop=True) + ) + + # 7) Write consolidated CSV + affils_output_path = Path(affils_output_path) + affils_output_path.parent.mkdir(parents=True, exist_ok=True) + if affils_output_path.suffix == "": + affils_output_path = affils_output_path.with_suffix(".csv") + aff_year_counts.to_csv(affils_output_path, index=False, encoding="utf-8-sig") + + print(f"Wrote {len(aff_year_counts)} rows to {affils_output_path} " + f"(≥ {min_total_papers} papers" + f"{', country=' + country_filter if country_filter is not None else ', all countries'})") + + # 8) Optional: one file per year (same columns) + if partition_by_year and not aff_year_counts.empty: + out_dir = Path(per_year_output_dir) if per_year_output_dir else affils_output_path.parent + out_dir.mkdir(parents=True, exist_ok=True) + stem = affils_output_path.stem + suffix = affils_output_path.suffix or ".csv" + for yr in sorted(aff_year_counts['year'].unique()): + yr_df = aff_year_counts[aff_year_counts['year'] == yr] + yr_path = out_dir / f"{stem}.year={yr}{suffix}" + yr_df.to_csv(yr_path, index=False, encoding="utf-8-sig") + print(f"→ Wrote {len(yr_df)} rows for year {yr} to {yr_path}") + +# # All countries in output (missing → 'unknown'), consolidated CSV only +# generate_top_affiliations_with_country( +# "papers.csv", "out/affiliations_top.csv", min_total_papers=20 +# ) + +# # Only the United States (others excluded), plus per-year files +# generate_top_affiliations_with_country( +# "papers.csv", "out/affiliations_top.csv", +# min_total_papers=10, +# country_filter="United States", +# partition_by_year=True, +# per_year_output_dir="out/by_year" +# ) + +# # Only entries whose country was missing in the source (now labeled 'unknown') +# generate_top_affiliations_with_country( +# "papers.csv", "out/affiliations_unknown.csv", +# min_total_papers=5, +# country_filter="unknown" +# ) diff --git a/TELF/pipeline/blocks/block_helpers/author_partition.py b/TELF/pipeline/blocks/block_helpers/author_partition.py new file mode 100644 index 00000000..9a6b2970 --- /dev/null +++ b/TELF/pipeline/blocks/block_helpers/author_partition.py @@ -0,0 +1,276 @@ +import ast +import json +import re +import pandas as pd + +def write_top_authors_by_cluster( + df_path: str, + output_path: str, + COUNTY_NAMES=None, # keep name to preserve identical print message + top_n: int = 10, + debug: bool = False # optional: prints row counts at key steps +): + """ + Runs the same pipeline but with robust parsing, flexible author extraction, + normalized country filtering, and sensible fallbacks. + Keeps columns, encoding, and print message identical to your original. + """ + def _debug(msg): + if debug: + print(msg) + + # ---------- helpers ---------- + _split_re = re.compile(r"[;,|]\s*") + + def _safe_eval_aff(s): + """Parse affiliations cell to dict-like; return {} on any issue.""" + if isinstance(s, dict): + return s + if isinstance(s, list): + return s + if pd.isna(s): + return {} + txt = str(s).strip() + if not txt: + return {} + # Try JSON first (handles true/false/null) + try: + return json.loads(txt) + except Exception: + pass + # Fallback to Python literal + try: + return ast.literal_eval(txt) + except Exception: + return {} + + def _listify_author_ids(value): + """Return a list[str] of author IDs from various shapes (list/json/csv).""" + if value is None or (isinstance(value, float) and pd.isna(value)): + return [] + # Already list-like? + if isinstance(value, (list, tuple, set)): + return [str(x) for x in value if str(x).strip()] + s = str(value).strip() + if not s: + return [] + # Try JSON array + try: + arr = json.loads(s) + if isinstance(arr, (list, tuple, set)): + return [str(x) for x in arr if str(x).strip()] + except Exception: + pass + # Delimited fallback + parts = _split_re.split(s) + return [p for p in parts if p] + + def _authors_from_affinfo(x, row_level_authors): + """ + Extract authors from an affiliation 'info' dict, falling back to row-level authors + if none are present for that affiliation. + """ + if not isinstance(x, dict): + return row_level_authors + for k in ("authors", "author_ids", "authorId", "author_id", "authorIds"): + if k in x and x[k] is not None: + ids_ = _listify_author_ids(x[k]) + if ids_: + return ids_ + # fallback: use the paper's row-level authors (prevents empty explode) + return row_level_authors + + def _normalize_aff_dict(aff): + """ + Accept dict-of-dicts OR list-of-dicts and return dict[str_id] -> dict(info). + """ + if isinstance(aff, dict): + return aff + if isinstance(aff, list): + out = {} + for item in aff: + if isinstance(item, dict): + aff_id = (item.get("id") or item.get("affiliation_id") or item.get("affiliationId") or + item.get("grid") or item.get("ror") or item.get("name") or "unknown") + out[str(aff_id)] = item + return out + return {} + + # ---------- load & standardize ---------- + df = pd.read_csv(df_path) + + if 'cluster' not in df.columns: + if 'Graph_Name' in df.columns: + df = df.copy() + df['cluster'] = df['Graph_Name'] + else: + raise KeyError("Expected a 'cluster' column; neither 'cluster' nor 'Graph_Name' found.") + + # Build author_name_map (author_id ↔ author_name) from row-level columns + if not {'author_ids', 'authors'}.issubset(df.columns): + # Create empty map to avoid KeyError; we'll still count by ID + author_name_map = pd.DataFrame(columns=['author_id', 'author_name']) + else: + author_name_map = ( + df[['author_ids', 'authors']] + .assign( + author_ids=lambda d: d['author_ids'].astype(str).apply(_listify_author_ids), + authors=lambda d: d['authors'].astype(str).apply(_listify_author_ids), + ) + .explode(['author_ids', 'authors']) + .rename(columns={'author_ids': 'author_id', 'authors': 'author_name'}) + .assign(author_id=lambda d: d['author_id'].astype(str)) + .drop_duplicates(['author_id', 'author_name']) + ) + + # ---------- explode affiliations -> (cluster, affiliation, country, author_id) ---------- + # Carry row-level author_ids to allow fallback when an affiliation lacks per-affiliation authors + base_cols = ['cluster', 'affiliations'] + if 'author_ids' in df.columns: + base_cols.append('author_ids') + if 'authors' in df.columns: + base_cols.append('authors') + + tmp = ( + df[base_cols] + .assign( + row_authors=lambda d: d.get('author_ids', pd.Series([None]*len(d))).apply(_listify_author_ids) + ) + .assign(aff_raw=lambda d: d['affiliations'].map(_safe_eval_aff)) + .assign(aff_dict=lambda d: d['aff_raw'].map(_normalize_aff_dict)) + .assign(aff_items=lambda d: d['aff_dict'].map(lambda x: list(x.items()))) + .explode('aff_items', ignore_index=True) + ) + + # If nothing at all parsed, make a single "Unknown" slot per row so we can still count + if tmp['aff_items'].isna().all(): + tmp = ( + df[['cluster']].copy() + .assign( + row_authors=lambda d: df.get('author_ids', pd.Series([None]*len(df))).apply(_listify_author_ids), + aff_items=[('unknown', {'name': 'Unknown', 'country': 'unknown'})] * len(df) + ) + .explode('aff_items', ignore_index=True) + ) + + auth_aff = ( + tmp + .dropna(subset=['aff_items']) + .assign( + affiliation_id=lambda d: d['aff_items'].map(lambda p: p[0]), + aff_info=lambda d: d['aff_items'].map(lambda p: p[1]), + ) + .assign( + affiliation_name=lambda d: d['aff_info'].map(lambda x: x.get('name') if isinstance(x, dict) else None), + country_raw=lambda d: d['aff_info'].map(lambda x: x.get('country') if isinstance(x, dict) else None), + ) + ) + + # Author IDs from affiliation info, with fallback to row-level authors + if 'row_authors' not in auth_aff.columns: + auth_aff['row_authors'] = [[]] * len(auth_aff) + auth_aff = auth_aff.assign( + author_ids_list=lambda d: d.apply( + lambda r: _authors_from_affinfo(r['aff_info'], r['row_authors']), axis=1 + ) + ).explode('author_ids_list', ignore_index=True) + + # If *still* empty, bail out with a zero-row CSV but keep the same columns + if auth_aff.empty: + out = pd.DataFrame(columns=[ + 'cluster', 'rank', 'author_name', 'author_id', 'affiliation_name', 'country' + ]) + out.to_csv(output_path, index=False, encoding="utf-8-sig") + print(f"Wrote {len(out)} rows (authors in {COUNTY_NAMES}) to {output_path}") + return out + + auth_aff = auth_aff.assign( + author_id=lambda d: d['author_ids_list'].astype(str), + country=lambda d: d['country_raw'].astype(str).str.strip().replace({'': 'unknown', 'None': 'unknown', 'nan': 'unknown'}) + ).merge(author_name_map, on='author_id', how='left') + + _debug(f"rows after aff explode: {len(auth_aff)}") + + # ---------- optional country filter (normalized) ---------- + if COUNTY_NAMES: + # Normalize both sides (casefold + strip) and handle common US aliases + aliases = { + 'us': {'us', 'usa', 'u.s.', 'u.s.a.', 'united states', 'united states of america', 'u.s.a'}, + } + def _norm_country(s): + s = (s or "").strip().casefold() + if s in aliases['us']: + return 'united states' + return s + + want = { _norm_country(x) for x in COUNTY_NAMES } + auth_aff = auth_aff.assign(_country_norm=auth_aff['country'].map(_norm_country)) + before = len(auth_aff) + auth_aff = auth_aff[auth_aff['_country_norm'].isin(want)].copy() + auth_aff.drop(columns=['_country_norm'], inplace=True) + _debug(f"country filter kept {len(auth_aff)} / {before} rows") + + # ---------- counts ---------- + counts = ( + auth_aff + .groupby(['cluster', 'author_id', 'author_name', 'affiliation_name', 'country'], dropna=False) + .size() + .reset_index(name='paper_count') + ) + + if counts.empty: + # Write an empty CSV with the right columns + out = pd.DataFrame(columns=[ + 'cluster', 'rank', 'author_name', 'author_id', 'affiliation_name', 'country' + ]) + out.to_csv(output_path, index=False, encoding="utf-8-sig") + print(f"Wrote {len(out)} rows (authors in {COUNTY_NAMES}) to {output_path}") + return out + + # ---------- top N per cluster ---------- + top_authors = ( + counts + .groupby('cluster', group_keys=False) + .apply(lambda g: g.nlargest(top_n, 'paper_count')) + .reset_index(drop=True) + ) + top_authors['rank'] = ( + top_authors + .groupby('cluster')['paper_count'] + .rank(method='first', ascending=False) + .astype(int) + ) + + # ---------- pivot cross-cluster ---------- + pivot = ( + counts + .pivot_table( + index=['author_id', 'author_name', 'affiliation_name', 'country'], + columns='cluster', + values='paper_count', + aggfunc='sum', + fill_value=0, + ) + .reindex(sorted(counts['cluster'].unique()), axis=1) + .astype(int) + .reset_index() + ) + + # ---------- merge & write ---------- + result = top_authors.merge( + pivot, + on=['author_id', 'author_name', 'affiliation_name', 'country'], + how='left' + ) + + cluster_cols = [c for c in pivot.columns if c not in ['author_id', 'author_name', 'affiliation_name', 'country']] + cols = [ + 'cluster', 'rank', + 'author_name', 'author_id', + 'affiliation_name', 'country', + ] + cluster_cols + + out = result[cols] + out.to_csv(output_path, index=False, encoding="utf-8-sig") + print(f"Wrote {len(result)} rows (authors in {COUNTY_NAMES}) to {output_path}") + return out diff --git a/TELF/pipeline/blocks/block_helpers/hnmfk_paths.py b/TELF/pipeline/blocks/block_helpers/hnmfk_paths.py new file mode 100644 index 00000000..98e627af --- /dev/null +++ b/TELF/pipeline/blocks/block_helpers/hnmfk_paths.py @@ -0,0 +1,51 @@ +# post_processing/Peacock/node_sources.py +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Iterator, Optional + + +from ....factorization import HNMFk # adjust relative path if needed + + +@dataclass(frozen=True) +class Node: + dir: Path + csv: Path + + +class NodeSource: + def iter_nodes(self) -> Iterator[Node]: + raise NotImplementedError + + +class HNMFkNodeSource(NodeSource): + def __init__(self, experiment_path: Path, *, only_existing_csv: bool = True) -> None: + self.experiment_path = Path(experiment_path).expanduser().resolve() + self.only_existing_csv = only_existing_csv + + def iter_nodes(self) -> Iterator[Node]: + model = HNMFk(experiment_name=str(self.experiment_path)) + model.load_model() + seen = set() + for node in model.traverse_nodes(): + node_dir = Path(node["node_save_path"]).resolve().parent + if node_dir in seen: + continue + seen.add(node_dir) + # prefer latest cluster file in that directory + csvs = sorted(node_dir.glob("cluster_for_k=*.csv"), key=lambda p: p.stat().st_mtime) + if csvs: + yield Node(dir=node_dir, csv=csvs[-1]) + elif not self.only_existing_csv: + # allow “expected” path even if it doesn’t exist yet + yield Node(dir=node_dir, csv=node_dir / "cluster_for_k=UNKNOWN.csv") + + +class GlobNodeSource(NodeSource): + def __init__(self, root: Path) -> None: + self.root = Path(root).expanduser().resolve() + + def iter_nodes(self) -> Iterator[Node]: + for csv in sorted(self.root.rglob("cluster_for_k=*.csv")): + yield Node(dir=csv.parent, csv=csv) diff --git a/TELF/pipeline/blocks/block_helpers/peacock_renderer.py b/TELF/pipeline/blocks/block_helpers/peacock_renderer.py new file mode 100644 index 00000000..8198cf18 --- /dev/null +++ b/TELF/pipeline/blocks/block_helpers/peacock_renderer.py @@ -0,0 +1,267 @@ +# post_processing/Peacock/peacock_renderer.py +from __future__ import annotations +from pathlib import Path +from typing import Dict, Optional, Sequence +import ast +import json +import numpy as np +import pandas as pd + +# Prefer local (same-package) imports; fall back if vendored +try: + from .Utility import aggregate_ostats + from .Plot import plot_heatmap, plot_bar, plot_scatter +except Exception: + from ....post_processing.Peacock.Utility import aggregate_ostats + from ....post_processing.Peacock.Plot import plot_heatmap, plot_bar, plot_scatter + +plot_hist = plot_bar +plot_scatter3D = plot_scatter + + +# Normalize affiliations to a Python-literal string that TELF can ast.literal_eval +def _normalize_aff_to_py_literal(v): + """ + Accept dict/list/JSON/string; return a *Python-literal* string (repr), + guaranteeing per-affiliation 'authors' (list) & 'country' (str). + """ + # 1) Parse to Python object + if isinstance(v, (dict, list)): + obj = v + elif isinstance(v, str): + s = v.strip() + if not s: + obj = [] + else: + try: + obj = ast.literal_eval(s) + except Exception: + try: + obj = json.loads(s) + except Exception: + obj = [] + else: + obj = [] + + # 2) Canonicalize to dict-of-dicts keyed by id + if isinstance(obj, list): + out = {} + for i, item in enumerate(obj): + if not isinstance(item, dict): + continue + key = str(item.get("id", item.get("affiliation_id", i))) + out[key] = { + **item, + "authors": item.get("authors", item.get("author_ids", [])) or [], + "country": item.get("country", "Unknown"), + } + obj = out + elif isinstance(obj, dict): + out = {} + for k, val in obj.items(): + if not isinstance(val, dict): + continue + val.setdefault("authors", val.get("author_ids", [])) + val.setdefault("country", "Unknown") + out[str(k)] = val + obj = out + else: + obj = {} + + # 3) Return as Python-literal string (NOT JSON) so ast.literal_eval works + return repr(obj) + + +class PeacockRenderer: + def __init__( + self, + *, + hist_stats: Sequence[str] = ("paper_count", "num_citations"), + hist_ylabels: Optional[Dict[str, str]] = None, + col_names: Optional[Dict[str, str]] = None, + affiliation_palette: Optional[Dict[str, str]] = None, + country: Optional[str] = None, + ) -> None: + self.hist_stats = tuple(hist_stats) + self.hist_ylabels = hist_ylabels or { + "paper_count": "Number of Papers", + "num_citations": "Number of Citations", + "attribution_percentage": "Attribution Percentage", + } + self.col_names = col_names or { + "id": "eid", + "authors": "slic_authors", + "author_ids": "slic_author_ids", + "affiliations": "slic_affiliations", + "funding": "funding", + "citations": "num_citations", + "references": "references", + } + self.affiliation_palette = affiliation_palette or {} + self.country = country + + # --- PNG→HTML fallback (skip Kaleido deps when not present) --- + def _png_or_html(self, make_plot_func, stem: Path, *args, **kwargs): + """ + Try PNG via Kaleido; if it fails, emit HTML (interactive). + """ + png_path = stem.with_suffix(".png") + html_path = stem.with_suffix(".html") + try: + make_plot_func(*args, interactive=False, fname=str(png_path), **kwargs) + except Exception: + fig = make_plot_func(*args, interactive=True, fname=None, **kwargs) + fig.write_html(str(html_path), include_plotlyjs="cdn") + + def render(self, df: pd.DataFrame, out_dir: Path) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + df = df.copy() + + # Column names + aff_col = self.col_names["affiliations"] + aut_col = self.col_names["authors"] + aid_col = self.col_names["author_ids"] + + # 1) Coerce authors / author_ids to *semicolon-separated strings* (what TELF expects) + def _to_list_any(x): + if x is None or (isinstance(x, float) and pd.isna(x)): + return [] + if isinstance(x, (list, tuple)): + return list(x) + if isinstance(x, str): + s = x.strip() + if not s: + return [] + # try literal (['a','b']) then JSON, else split on ; or , + for parser in (ast.literal_eval, json.loads): + try: + v = parser(s) + if isinstance(v, list): + return v + except Exception: + pass + sep = ";" if ";" in s else "," + return [t.strip() for t in s.split(sep) if t.strip()] + return [str(x)] + + def _to_sc_str_preserve_nan(x): + lst = _to_list_any(x) + vals = [str(v).strip() for v in lst if str(v).strip() != ""] + return np.nan if not vals else ";".join(vals) + + df[aut_col] = df[aut_col].apply(_to_sc_str_preserve_nan) + df[aid_col] = df[aid_col].apply(_to_sc_str_preserve_nan) + + # 2) Basic filtering & year cast + # Keep NaNs so dropna can remove bad rows + subset = [self.col_names["id"], aut_col, aid_col, aff_col] + df = df.dropna(subset=subset) + + if "year" in df.columns: + df["year"] = pd.to_numeric(df["year"], errors="coerce") + df = df.dropna(subset=["year"]) + df["year"] = df["year"].astype(int) + else: + # If no year present, supply a dummy year so downstream heatmaps don't crash + df["year"] = 0 + + if df.empty: + # still emit empty top lists; skip plots + (out_dir / "top_authors.csv").write_text("") + (out_dir / "top_affiliations.csv").write_text("") + return + + # 3) Normalize affiliations to a Python-literal string for TELF's ast.literal_eval + df[aff_col] = df[aff_col].apply(_normalize_aff_to_py_literal) + + # Optional filters + filters = {"country": self.country} if self.country else None + + # Helper: safe pivot_table (handles duplicate index) + def _safe_pivot_table(data, index, columns, values): + if data is None or len(data) == 0: + return pd.DataFrame() + return data.pivot_table(index=index, columns=columns, values=values, aggfunc="sum", fill_value=0) + + # 4) Top-100 CSVs + author_stats = aggregate_ostats(df, key="author_id", top_n=100, col_names=self.col_names, filters=filters, by_year=False) + affiliation_stats = aggregate_ostats(df, key="affiliation_id", top_n=100, col_names=self.col_names, filters=filters, by_year=False) + author_stats.to_csv(out_dir / "top_authors.csv", index=False) + affiliation_stats.to_csv(out_dir / "top_affiliations.csv", index=False) + + # 5) Common args for top-10 by citations + auth_args = dict(key="author_id", top_n=10, sort_by="num_citations", col_names=self.col_names, by_year=True, filters=filters) + aff_args = dict(key="affiliation_id", top_n=10, sort_by="num_citations", col_names=self.col_names, by_year=True, filters=filters) + + # 6) Heatmaps (use pivot_table to tolerate duplicates) + auth_heat = aggregate_ostats(df, **auth_args) + aff_heat = aggregate_ostats(df, **aff_args) + + pivot_c = _safe_pivot_table(auth_heat, index="year", columns="author", values="num_citations") + pivot_p = _safe_pivot_table(auth_heat, index="year", columns="author", values="paper_count") + pivot_c2 = _safe_pivot_table(aff_heat, index="year", columns="affiliation", values="num_citations") + pivot_p2 = _safe_pivot_table(aff_heat, index="year", columns="affiliation", values="paper_count") + + if not pivot_c.empty: + self._png_or_html( + plot_heatmap, out_dir / "author_heatmap_citations", + pivot_c, cmap="jet", interpolation="gaussian", + title="Author Citations by Year", xlabel="Author", ylabel="Year" + ) + if not pivot_p.empty: + self._png_or_html( + plot_heatmap, out_dir / "author_heatmap_papers", + pivot_p, cmap="jet", interpolation="gaussian", + title="Author Papers by Year", xlabel="Author", ylabel="Year" + ) + if not pivot_c2.empty: + self._png_or_html( + plot_heatmap, out_dir / "affiliation_heatmap_citations", + pivot_c2, cmap="jet", interpolation="gaussian", + title="Affiliation Citations by Year", xlabel="Affiliation", ylabel="Year" + ) + if not pivot_p2.empty: + self._png_or_html( + plot_heatmap, out_dir / "affiliation_heatmap_papers", + pivot_p2, cmap="jet", interpolation="gaussian", + title="Affiliation Papers by Year", xlabel="Affiliation", ylabel="Year" + ) + + # 7) Histograms + auth_hist = aggregate_ostats(df, **{**auth_args, "by_year": False}) + if not auth_hist.empty: + self._png_or_html( + plot_hist, out_dir / "author_hist", + auth_hist, x="author", ys=list(self.hist_stats), + title="Author Statistics Histogram", xlabel="Author", + ylabel=self.hist_ylabels[self.hist_stats[0]] + ) + + aff_hist = aggregate_ostats(df, **{**aff_args, "by_year": False}) + if not aff_hist.empty: + self._png_or_html( + plot_hist, out_dir / "affiliation_hist", + aff_hist, x="affiliation", ys=list(self.hist_stats), + title="Affiliation Statistics Histogram", xlabel="Affiliation", + ylabel=self.hist_ylabels[self.hist_stats[0]] + ) + + # 8) 3D scatter (safe to call even on smaller slices) + self._png_or_html( + plot_scatter3D, out_dir / "author_scatter", + df, x="paper_count", y="attribution_percentage", z="num_citations", + agg_func=aggregate_ostats, agg_kwargs=auth_args, + log_z=True, hue="affiliation", labels="author", + title="Author Stats Scatter3D", xlabel="Paper Count", + ylabel="Attribution Percentage", zlabel="Num. Citations", + base_palette=self.affiliation_palette, + ) + self._png_or_html( + plot_scatter3D, out_dir / "affiliation_scatter", + df, x="paper_count", y="attribution_percentage", z="num_citations", + agg_func=aggregate_ostats, agg_kwargs=aff_args, + log_z=True, hue="country", labels="affiliation", + title="Affiliation Stats Scatter3D", xlabel="Paper Count", + ylabel="Attribution Percentage", zlabel="Num. Citations", + base_palette=self.affiliation_palette, + ) diff --git a/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py b/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py index 9f33b617..70e0712c 100644 --- a/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py +++ b/TELF/pipeline/blocks/collect_hnmfk_leaf_block.py @@ -51,7 +51,7 @@ def __init__( self, *, needs: Sequence[str] = CANONICAL_NEEDS, - provides: Sequence[str] = ("leaf_data_csv", "leaf_labels_csv"), + provides: Sequence[str] = ("df", "leaf_labels_csv"), tag: str = "LeafDataLabels", init_settings: Optional[Dict[str, Any]] = None, call_settings: Optional[Dict[str, Any]] = None, @@ -427,9 +427,9 @@ def visit(node_name: str): self._log(log_fp, f"Failed to write summary.txt: {e}") # Checkpoint + bundle exposure - self.register_checkpoint("leaf_data_csv", leaf_data_csv) + self.register_checkpoint("df", leaf_data_csv) self.register_checkpoint("leaf_labels_csv", leaf_labels_csv) - bundle[f"{self.tag}.leaf_data_csv"] = leaf_data_csv + bundle[f"{self.tag}.df"] = leaf_data_csv bundle[f"{self.tag}.leaf_labels_csv"] = leaf_labels_csv # Optional: also expose summary path in the bundle (no need to register) bundle[f"{self.tag}.summary_txt"] = summary_txt # NEW @@ -444,4 +444,4 @@ def visit(node_name: str): f"wrote summary at {summary_txt}", ) if self.verbose: - print(f"[{self.tag}] Wrote:\n {leaf_data_csv}\n {leaf_labels_csv}\n {summary_txt}") + print(f"[{self.tag}] Wrote:\n {leaf_data_csv}\n {leaf_labels_csv}\n {summary_txt}") \ No newline at end of file diff --git a/TELF/pipeline/blocks/peacock_stats_block.py b/TELF/pipeline/blocks/peacock_stats_block.py index db9591f8..af35f22b 100644 --- a/TELF/pipeline/blocks/peacock_stats_block.py +++ b/TELF/pipeline/blocks/peacock_stats_block.py @@ -1,239 +1,119 @@ # pipeline/blocks/peacock_stats_block.py from __future__ import annotations from pathlib import Path -from typing import Any, Dict, Sequence, Optional, Tuple - +from typing import Any, Dict, Optional, Sequence, Tuple, Literal import pandas as pd from .base_block import AnimalBlock -from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY, RESULTS_DEFAULT -# Peacock aggregation function -from ...post_processing.Peacock.Utility import aggregate_ostats +from .block_helpers.peacock_renderer import PeacockRenderer +from .block_helpers.hnmfk_paths import NodeSource, HNMFkNodeSource, GlobNodeSource -# Peacock plotting functions (the ones you pasted from Plot/plot.py) -from ...post_processing.Peacock.Plot import ( - plot_heatmap, - plot_bar, - plot_scatter, -) -# Convenience aliases -plot_hist = plot_bar -plot_scatter3D = plot_scatter +Mode = Literal["single", "hnmfk", "glob"] class PeacockStatsBlock(AnimalBlock): - CANONICAL_NEEDS: Tuple[str, ...] = ("df", SAVE_DIR_BUNDLE_KEY) + CANONICAL_NEEDS: Tuple[str, ...] = ("df", ) def __init__( self, *, needs: Sequence[str] = CANONICAL_NEEDS, provides: Sequence[str] = ("outpath",), + mode: Mode = "single", + # renderer args hist_stats: Sequence[str] = ("paper_count", "num_citations"), hist_ylabels: Optional[Dict[str, str]] = None, col_names: Optional[Dict[str, str]] = None, affiliation_palette: Optional[Dict[str, str]] = None, country: Optional[str] = None, + # discovery args + experiment_path: Optional[str] = None, # for mode="hnmfk" + glob_root: Optional[str] = None, # for mode="glob" + skip_completed: bool = True, **kw: Any, ) -> None: + self.mode = mode + self.experiment_path = Path(experiment_path).expanduser().resolve() if experiment_path else None + self.glob_root = Path(glob_root).expanduser().resolve() if glob_root else None + self.skip_completed = skip_completed + + self.renderer = PeacockRenderer( + hist_stats=hist_stats, + hist_ylabels=hist_ylabels, + col_names=col_names, + affiliation_palette=affiliation_palette, + country=country, + ) + + # require model_path if mode==hnmfk (via conditional_needs), but keep simple API + conds = list(kw.pop("conditional_needs", ())) + if self.mode == "hnmfk" and not self.experiment_path: + conds.append(("model_path", lambda _b, _s: True)) + super().__init__( needs=needs, provides=provides, tag="PeacockStats", init_settings={}, call_settings={}, + conditional_needs=tuple(conds), **kw, ) - self.hist_stats = tuple(hist_stats) - self.hist_ylabels = hist_ylabels or { - "paper_count": "Number of Papers", - "num_citations": "Number of Citations", - "attribution_percentage": "Attribution Percentage", - } - self.col_names = col_names or { - "id": "eid", - "authors": "slic_authors", - "author_ids": "slic_author_ids", - "affiliations": "slic_affiliations", - "funding": "funding", - "citations": "num_citations", - "references": "references", - } - self.affiliation_palette = affiliation_palette or {} - self.country = country + # —————————————————————————————————————————— + def _source(self, bundle: DataBundle) -> Optional[NodeSource]: + if self.mode == "hnmfk": + exp = self.experiment_path or Path(str(bundle["model_path"])) + return HNMFkNodeSource(exp) + if self.mode == "glob": + root = self.glob_root or Path(bundle.get(SAVE_DIR_BUNDLE_KEY, RESULTS_DEFAULT)) + return GlobNodeSource(root) + return None # single mode + + def _parent_out_dir(self, bundle: DataBundle) -> Path: + # where the block-level checkpoint lands if needed + base = Path(bundle.get(SAVE_DIR_BUNDLE_KEY, RESULTS_DEFAULT)) + return base / self.tag + + # —————————————————————————————————————————— def run(self, bundle: DataBundle) -> None: - # 1) Inputs & cleanup - df: pd.DataFrame = bundle["df"] - # Save everything under the block's tag directory (like other blocks) - root_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) - out_dir = root_dir / self.tag - out_dir.mkdir(parents=True, exist_ok=True) - - # ensure affiliation column is string - aff_col = self.col_names["affiliations"] - df[aff_col] = df[aff_col].apply(lambda x: x if isinstance(x, str) else str(x)) - - df = ( - df - .dropna(subset=[ - self.col_names["id"], - self.col_names["authors"], - self.col_names["author_ids"], - aff_col, - ]) - .assign(year=lambda d: d.year.astype(int)) - ) - - filters = {"country": self.country} if self.country else None - - # 2) Write top‐100 CSVs - author_stats = aggregate_ostats(df, key="author_id", top_n=100, col_names=self.col_names, filters=filters, by_year=False) - affiliation_stats = aggregate_ostats(df, key="affiliation_id", top_n=100, col_names=self.col_names, filters=filters, by_year=False) - author_stats.to_csv(out_dir / "top_authors.csv", index=False) - affiliation_stats.to_csv(out_dir / "top_affiliations.csv", index=False) - - # 3) Shared “top‐10 by citations” args - auth_args = dict( - key="author_id", - top_n=10, - sort_by="num_citations", - col_names=self.col_names, - by_year=True, - filters=filters, - ) - aff_args = dict( - key="affiliation_id", - top_n=10, - sort_by="num_citations", - col_names=self.col_names, - by_year=True, - filters=filters, - ) - - # 4) Heatmaps — pivot then call plot_heatmap - # ─ Authors, citations - auth_heat = aggregate_ostats(df, **auth_args) - pivot_c = auth_heat.pivot(index="year", columns="author", values="num_citations").fillna(0) - plot_heatmap( - pivot_c, - cmap="jet", - interpolation="gaussian", - fname=str(out_dir / "author_heatmap_citations.png"), - interactive=False, - title="Author Citations by Year", - xlabel="Author", - ylabel="Year", - ) - # ─ Authors, papers - pivot_p = auth_heat.pivot(index="year", columns="author", values="paper_count").fillna(0) - plot_heatmap( - pivot_p, - cmap="jet", - interpolation="gaussian", - fname=str(out_dir / "author_heatmap_papers.png"), - interactive=False, - title="Author Papers by Year", - xlabel="Author", - ylabel="Year", - ) - - # ─ Affiliations, citations - aff_heat = aggregate_ostats(df, **aff_args) - pivot_c2 = aff_heat.pivot(index="year", columns="affiliation", values="num_citations").fillna(0) - plot_heatmap( - pivot_c2, - cmap="jet", - interpolation="gaussian", - fname=str(out_dir / "affiliation_heatmap_citations.png"), - interactive=False, - title="Affiliation Citations by Year", - xlabel="Affiliation", - ylabel="Year", - ) - # ─ Affiliations, papers - pivot_p2 = aff_heat.pivot(index="year", columns="affiliation", values="paper_count").fillna(0) - plot_heatmap( - pivot_p2, - cmap="jet", - interpolation="gaussian", - fname=str(out_dir / "affiliation_heatmap_papers.png"), - interactive=False, - title="Affiliation Papers by Year", - xlabel="Affiliation", - ylabel="Year", - ) - - # 5) Histograms (bar‐plots) - auth_hist = aggregate_ostats(df, **{**auth_args, "by_year": False}) - plot_hist( - auth_hist, - x="author", - ys=list(self.hist_stats), - fname=str(out_dir / "author_hist.png"), - interactive=False, - # cmap="husl", # ← remove this line... - title="Author Statistics Histogram", - xlabel="Author", - ylabel=self.hist_ylabels[self.hist_stats[0]], - ) - aff_hist = aggregate_ostats(df, **{**aff_args, "by_year": False}) - plot_hist( - aff_hist, - x="affiliation", - ys=list(self.hist_stats), - fname=str(out_dir / "affiliation_hist.png"), - interactive=False, - # cmap="husl", - title="Affiliation Statistics Histogram", - xlabel="Affiliation", - ylabel=self.hist_ylabels[self.hist_stats[0]], - ) - - # 6) 3-D scatter - plot_scatter3D( - df, - x="paper_count", - y="attribution_percentage", - z="num_citations", - agg_func=aggregate_ostats, - agg_kwargs=auth_args, - fname=str(out_dir / "author_scatter.png"), - interactive=False, - log_z=True, - hue="affiliation", - labels="author", - title="Author Stats Scatter3D", - xlabel="Paper Count", - ylabel="Attribution Percentage", - zlabel="Num. Citations", - base_palette=self.affiliation_palette, - ) - plot_scatter3D( - df, - x="paper_count", - y="attribution_percentage", - z="num_citations", - agg_func=aggregate_ostats, - agg_kwargs=aff_args, - fname=str(out_dir / "affiliation_scatter.png"), - interactive=False, - log_z=True, - hue="country", - labels="affiliation", - title="Affiliation Stats Scatter3D", - xlabel="Paper Count", - ylabel="Attribution Percentage", - zlabel="Num. Citations", - base_palette=self.affiliation_palette, - ) - - # 7) Dummy checkpoint under your single `provides` key - if SAVE_DIR_BUNDLE_KEY in bundle: - final_csv = out_dir / "none.csv" - final_csv.write_text("") # empty placeholder - self.register_checkpoint(self.provides[0], final_csv) - bundle[f"{self.tag}.{self.provides[0]}"] = final_csv - # (no return: AnimalBlock.__call__ returns the bundle) + # mode: single (unchanged behavior) + if self.mode == "single": + df: pd.DataFrame = bundle["df"] + out_dir = Path(bundle.get(SAVE_DIR_BUNDLE_KEY, RESULTS_DEFAULT)) + self.renderer.render(df, out_dir) + ckpt_dir = self._parent_out_dir(bundle); ckpt_dir.mkdir(parents=True, exist_ok=True) + marker = ckpt_dir / "none.csv"; marker.write_text("status\nok\n") + self.register_checkpoint(self.provides[0], marker) + bundle[f"{self.tag}.{self.provides[0]}"] = marker + return + + # mode: per-node (hnmfk or glob) + source = self._source(bundle) + assert source is not None, "Invalid configuration: per-node mode requires a NodeSource." + + produced = [] + for node in source.iter_nodes(): + out_dir = node.dir / "peacock" + if self.skip_completed and (out_dir / "PeacockStats.done").exists(): + produced.append(out_dir) + continue + + if not node.csv.exists(): + # skip nodes that haven't been post-processed yet + continue + + df_local = pd.read_csv(node.csv) + self.renderer.render(df_local, out_dir) + (out_dir / "PeacockStats.done").write_text("ok") + produced.append(out_dir) + + # block-level checkpoint + ckpt_dir = self._parent_out_dir(bundle); ckpt_dir.mkdir(parents=True, exist_ok=True) + registry = ckpt_dir / "per_node_registry.txt" + registry.write_text("\n".join(map(str, produced))) + self.register_checkpoint(self.provides[0], registry) + bundle[f"{self.tag}.{self.provides[0]}"] = registry diff --git a/TELF/pipeline/blocks/spacey_NER_block.py b/TELF/pipeline/blocks/spacey_NER_block.py index 862afcbf..8c70e955 100644 --- a/TELF/pipeline/blocks/spacey_NER_block.py +++ b/TELF/pipeline/blocks/spacey_NER_block.py @@ -10,36 +10,37 @@ class SpacyNERBlock(AnimalBlock): """ spaCy NER over one or more text columns (default: ['title', 'abstract']). - For each specified text column `col`, adds: - - f"{col}_ents": JSON list of dicts {text, label, start, end} - - f"{col}_ents_by_label": JSON dict of {label: [unique entity strings]} + Adds ONE new column to the DataFrame: + - `output_column` (default: 'ner_by_label'): a STRING of a dictionary that + can be round-tripped with `ast.literal_eval`. The dict maps: + { : [unique entity strings for the row] } Artifacts: - - /spaceyNER/spaceyNER.csv (enriched DataFrame) - - /spaceyNER/entities.csv (optional, exploded entity rows) + - //.csv (enriched DataFrame) Requirements: - - spaCy with a model installed (default: 'en_core_web_sm'). + - spaCy with a model installed (default: 'en_core_web_lg'). Parameters ---------- needs : tuple Bundle keys to read. Default: ("df",). provides : tuple - Bundle keys to write. Default: ("df", "ents_table"). - If you pass only ("df",), the exploded entities table is skipped. + Bundle keys to write. Default: ("df",). text_columns : Optional[List[str]] Which DF columns to run NER on. Default: ["title", "abstract"]. id_field : str - Row identifier, used in the exploded entities table. Default: "eid". + (Unused for output but kept for compatibility.) Default: "eid". spacy_model : str - spaCy model name to load. Default: "en_core_web_sm". + spaCy model name to load. Default: "en_core_web_lg". batch_size : int spaCy pipe batch size. Default: 256. n_process : int Number of processes for spaCy pipe. Default: 1. drop_existing : bool - If True, drop and rebuild any existing NER output columns. Default: True. + If True, drop an existing output column before writing. Default: True. + output_column : str + Name of the new column to write. Default: "ner_by_label". tag : str Block tag and artifact folder name. Default: "spaceyNER". init_settings : Optional[Dict[str, Any]] @@ -52,13 +53,14 @@ def __init__( self, *, needs=CANONICAL_NEEDS, - provides=("df", "ents_table"), + provides=("df",), text_columns: Optional[List[str]] = None, id_field: str = "eid", spacy_model: str = "en_core_web_lg", batch_size: int = 256, n_process: int = 1, drop_existing: bool = True, + output_column: str = "ner_by_label", tag: str = "spaceyNER", init_settings: Optional[Dict[str, Any]] = None, **kw, @@ -71,6 +73,7 @@ def __init__( "batch_size": int(batch_size), "n_process": int(n_process), "drop_existing": bool(drop_existing), + "output_column": output_column, "verbose": True, } @@ -131,83 +134,46 @@ def run(self, bundle: DataBundle) -> None: batch_size = int(self.init_settings.get("batch_size", 256)) n_process = int(self.init_settings.get("n_process", 1)) drop_existing = bool(self.init_settings.get("drop_existing", True)) + out_col = str(self.init_settings.get("output_column", "ner_by_label")) - # 4) Prepare destination columns; optionally drop existing - dest_cols = [] - for col in present: - dest_cols += [f"{col}_ents", f"{col}_ents_by_label"] - if drop_existing: - to_drop = [c for c in dest_cols if c in df.columns] - if to_drop: - df = df.drop(columns=to_drop) - print(f"[{self.tag}] dropped existing NER columns: {to_drop}") - + # 4) Prepare DF; optionally drop existing output column df_proc = df.copy() - exploded_rows: List[Dict[str, Any]] = [] + if drop_existing and out_col in df_proc.columns: + df_proc = df_proc.drop(columns=[out_col]) + print(f"[{self.tag}] dropped existing column: {out_col}") - # 5) Run NER column-by-column + # We'll aggregate per-row across ALL present text columns. + # For each row, maintain a dict[label] -> list[str] (unique, preserve insertion order) + num_rows = len(df_proc) + agg_by_row: List[Dict[str, List[str]]] = [dict() for _ in range(num_rows)] + + # 5) Run NER column-by-column and merge results per row for col in present: print(f"[{self.tag}] NER on column = {col}") texts = df_proc[col].fillna("").astype(str).tolist() - ents_json_col: List[str] = [] - bylabel_json_col: List[str] = [] - - i_to_row_id = ( - df_proc[self.id_field].tolist() if self.id_field in df_proc.columns else list(range(len(df_proc))) - ) for i, doc in enumerate(nlp.pipe(texts, batch_size=batch_size, n_process=n_process)): - ents = [ - { - "text": ent.text, - "label": ent.label_, - "start": int(ent.start_char), - "end": int(ent.end_char), - } - for ent in doc.ents - ] - - by_label: Dict[str, List[str]] = {} + if not doc.ents: + continue + row_dict = agg_by_row[i] for ent in doc.ents: - lst = by_label.setdefault(ent.label_, []) - if ent.text not in lst: - lst.append(ent.text) + label = ent.label_ + text = ent.text + lst = row_dict.setdefault(label, []) + # preserve insertion order & uniqueness + if text not in lst: + lst.append(text) - ents_json_col.append(json.dumps(ents, ensure_ascii=False)) - bylabel_json_col.append(json.dumps(by_label, ensure_ascii=False)) + # 6) Serialize each row's dict to a string compatible with ast.literal_eval + # Using JSON ensures safe, unambiguous formatting; ast.literal_eval accepts JSON literals. + serialized = [json.dumps(d, ensure_ascii=False) for d in agg_by_row] - # Collect exploded rows - rid = i_to_row_id[i] - for ent in doc.ents: - exploded_rows.append( - { - self.id_field: rid, - "source_column": col, - "text": ent.text, - "label": ent.label_, - "start": int(ent.start_char), - "end": int(ent.end_char), - } - ) - - df_proc[f"{col}_ents"] = ents_json_col - df_proc[f"{col}_ents_by_label"] = bylabel_json_col - - # 6) Save artifacts + register + df_proc[out_col] = serialized + + # 7) Save artifacts + register df_path = out / f"{self.tag}.csv" df_proc.to_csv(df_path, index=False, encoding="utf-8-sig") self.register_checkpoint(self.provides[0], df_path) bundle[f"{self.tag}.{self.provides[0]}"] = df_proc print(f"[{self.tag}] saved df → {df_path}") - - # optional exploded entities table - if len(self.provides) > 1: - ents_df = pd.DataFrame( - exploded_rows, - columns=[self.id_field, "source_column", "text", "label", "start", "end"], - ) - table_path = out / "entities.csv" - ents_df.to_csv(table_path, index=False, encoding="utf-8-sig") - self.register_checkpoint(self.provides[1], table_path) - bundle[f"{self.tag}.{self.provides[1]}"] = ents_df - print(f"[{self.tag}] saved entities → {table_path}") + print(f"[{self.tag}] added column = {out_col}") \ No newline at end of file diff --git a/TELF/pipeline/blocks/termite_neo4j_block.py b/TELF/pipeline/blocks/termite_neo4j_block.py index 5f756d4f..1a68008b 100644 --- a/TELF/pipeline/blocks/termite_neo4j_block.py +++ b/TELF/pipeline/blocks/termite_neo4j_block.py @@ -3,14 +3,17 @@ import os from pathlib import Path -from typing import Any, Dict, Sequence, Tuple, Optional +from typing import Any, Dict, Sequence, Tuple, Optional, List import pandas as pd from copy import deepcopy import ast +import json +import re + from .base_block import AnimalBlock from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY -# --- Termite + constants (as in your notebook) --- +# --- Termite + constants --- from TELF.applications import Termite from TELF.applications.Termite.neo4j_termite import ( ENTITY, RETURN_TYPE, ATTRIBUTES, ET, YEAR_TYPE, FROM_COL, DOCUMENT_TYPE, @@ -23,297 +26,547 @@ AFFILIATION_COUNTRY_RELATION, DOCUMENT_CATEGORY_RELATION, DOCUMENT_ACRONYM_RELATION ) -# ---------- helpers (exactly your notebook’s logic) ---------- +# --------------------------- NER labels +NER_LABELS = [ + "ORG", "PERSON", "GPE", "NORP", "FAC", "LOC", "PRODUCT", + "EVENT", "WORK_OF_ART", "LAW", "LANGUAGE", + "DATE", "TIME", "PERCENT", "MONEY", "QUANTITY", "ORDINAL", "CARDINAL", +] + +# ====================================================================================== +# Helpers +# ====================================================================================== + +def _get(row, key, default=None): + """Accessor that works for Series/dict/object.""" + try: + if key in row: + return row[key] + except Exception: + pass + if hasattr(row, key): + return getattr(row, key) + if hasattr(row, "get"): + try: + return row.get(key, default) + except Exception: + pass + return default + def list_split_no_attrs(data, split_with=';'): - returnable_entities = [] + out = [] if isinstance(data, str): - for entity_value in data.split(split_with): - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = entity_value - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] + for v in data.split(split_with): + e = deepcopy(RETURN_TYPE) + e[ENTITY] = v.strip() + out.append(e) + return out + return [deepcopy(RETURN_TYPE)] def get_cites(args): - data_string = args['data'] - return list_split_no_attrs(data_string.citations) + return list_split_no_attrs(args['data'].citations) def get_cited(args): - data_string = args['data'] - return list_split_no_attrs(data_string.references) + return list_split_no_attrs(args['data'].references) def get_authors_ID(args): - data_string = args['data'] - data = data_string.s2_author_ids + row = args['data'] + data = row.s2_author_ids if isinstance(data, str): - split_author_ids = data.split(';') - authors = data_string.s2_authors - if isinstance(authors, str): - authors_split_values = authors.split(';') - else: - authors_split_values = [] - returnable_entities = [] - for entity_value, attribute in zip(split_author_ids, authors_split_values): - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = entity_value - entity_returnable[ATTRIBUTES] = [('name', attribute)] - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] - + ids = data.split(';') + authors = row.s2_authors if isinstance(row.s2_authors, str) else '' + names = authors.split(';') if authors else [] + out = [] + for i, name in zip(ids, names): + e = deepcopy(RETURN_TYPE) + e[ENTITY] = i + e[ATTRIBUTES] = [('name', name)] + out.append(e) + return out + return [deepcopy(RETURN_TYPE)] def get_parent_topic(args): - data_string = args['data'] - returnable_entities = [] - parent_name = data_string.parent_name - if type(parent_name) == str: - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = parent_name - entity_returnable['attributes'] = [("Graph_Name",parent_name)] - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] - + data = args['data'] + graph_name = _get(data, 'Graph_Name') or _get(data, 'graph_name') + if not isinstance(graph_name, str) or not graph_name.strip(): + return [] + gn = graph_name.strip() + parent = gn.rsplit('_', 1)[0] if '_' in gn else None + if not parent: + return [] + e = deepcopy(RETURN_TYPE) + e[ENTITY] = parent + e[ATTRIBUTES] = {"entity_type": "Topic"} + return [e] + def get_topic_keywords(args): - data_string = args['data'] - returnable_entities = [] - keyword_list = data_string.words - if isinstance(keyword_list, str): - keyword_list = ast.literal_eval(keyword_list) - for keyword in keyword_list: - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = keyword - returnable_entities.append(entity_returnable) - if returnable_entities: - return returnable_entities + data = args['data'] + raw = _get(data, 'keywords') or _get(data, 'words') or _get(data, 'keyword_list') + + items: List[str] + if raw is None: + items = [] + elif isinstance(raw, (list, tuple, set)): + items = list(raw) else: - return [deepcopy(RETURN_TYPE)] + s = str(raw).strip() + if not s: + items = [] + else: + try: + maybe = ast.literal_eval(s) + if isinstance(maybe, (list, tuple, set)): + items = list(maybe) + else: + items = [maybe] + except Exception: + items = re.split(r'[,\|;]\s*', s) + + seen = set() + keywords = [] + for w in items: + w = str(w).strip() + if w and w not in seen: + seen.add(w) + keywords.append(w) + + out = [] + for w in keywords: + e = deepcopy(RETURN_TYPE) + e[ENTITY] = w + e[ATTRIBUTES] = {"entity_type": "Keyword"} + out.append(e) + return out def get_affiliations(args): - returnable_entities = [] - affil_string = args['data'].affiliations - if type(affil_string) == str and affil_string != 'nan': - for k, v in ast.literal_eval(affil_string).items(): + out = [] + s = args['data'].affiliations + if isinstance(s, str) and s != 'nan': + for k, v in ast.literal_eval(s).items(): if isinstance(v, dict): - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = k - name = v['name'] - entity_returnable[ATTRIBUTES] = [('name', name)] - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] - + e = deepcopy(RETURN_TYPE) + e[ENTITY] = k + e[ATTRIBUTES] = [('name', v.get('name'))] + out.append(e) + return out + return [deepcopy(RETURN_TYPE)] + def get_countries(args): - data_string = args['data'] - returnable_entities = [] - affil_string = data_string.affiliations - if type(affil_string) == str and affil_string != 'nan': - for k, v in ast.literal_eval(data_string.affiliations).items(): + row = args['data'] + s = row.affiliations + out = [] + if isinstance(s, str) and s != 'nan': + for _, v in ast.literal_eval(s).items(): if isinstance(v, dict): - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = v['country'] - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] + e = deepcopy(RETURN_TYPE) + e[ENTITY] = v.get('country') + out.append(e) + return out + return [deepcopy(RETURN_TYPE)] def get_categories(args): - data_string = args['data'] - returnable_entities = [] - subject_areas = data_string.subject_areas - if type(subject_areas) == str: - split_subjects= subject_areas.split(';') - for subject in split_subjects: - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = subject - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] - -def split_string(args, split_with= ';'): - data_string = args['data'] - return data_string.split(split_with) - -def list_split_no_attrs(data, split_with=';'): - returnable_entities = [] - if type(data) == str: - split_values = data.split(split_with) - for entity_value in split_values: - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = entity_value.strip() - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] - -def get_authors_ID(args): - data_string = args['data'] - data = data_string.author_ids - if type(data) == str: - split_author_ids= data.split(';') - authors = data_string.authors - if type(authors) == str: - authors_split_values = authors.split(';') - returnable_entities = [] - for entity_value, attribute in zip(split_author_ids, authors_split_values ): - entity_returnable = deepcopy(RETURN_TYPE) - entity_returnable[ENTITY] = entity_value - entity_returnable[ATTRIBUTES] = [('name', attribute)] - returnable_entities.append(entity_returnable) - return returnable_entities - else: - return [deepcopy(RETURN_TYPE)] + row = args['data'] + sa = row.subject_areas + if isinstance(sa, str): + out = [] + for subj in sa.split(';'): + e = deepcopy(RETURN_TYPE) + e[ENTITY] = subj.strip() + out.append(e) + return out + return [deepcopy(RETURN_TYPE)] + +def split_string(args, split_with=';'): + return args['data'].split(split_with) + +def get_authors_ID_simple(args): + row = args['data'] + data = row.author_ids + if isinstance(data, str): + ids = data.split(';') + authors = row.authors if isinstance(row.authors, str) else '' + names = authors.split(';') if authors else [] + out = [] + for i, name in zip(ids, names): + e = deepcopy(RETURN_TYPE) + e[ENTITY] = i + e[ATTRIBUTES] = [('name', name)] + out.append(e) + return out + return [deepcopy(RETURN_TYPE)] -# def get_acronyms(args): -# data_string = args['data'] -# return list_split_no_attrs(data_string.acronym_attribution, split_with=', ') def get_acronyms(args): - """ - Extract acronym strings from a row, tolerating missing columns. - Tries columns in order: 'acronym_attribution', 'acronyms', 'acronym'. - Returns [] when nothing is present so no triples are created. - """ + """Tries columns: 'acronym_attribution', 'acronyms', 'acronym'.""" row = args.get('data', None) if row is None: return [] - - candidates = ('acronym_attribution', 'acronyms', 'acronym') - - def _get_from_series(r, key): - try: - # pandas Series: prefer dict-style to avoid AttributeError when missing - if hasattr(r, 'get'): - return r.get(key, None) - # fallback for objects with attributes - return getattr(r, key, None) - except Exception: - return None - - value = None - for col in candidates: - v = _get_from_series(row, col) + for col in ('acronym_attribution', 'acronyms', 'acronym'): + v = _get(row, col) if v is not None and str(v).strip() and str(v).lower() != 'nan': - value = v - break - - if not value: - return [] # no acronyms -> no triples - - # Accept either comma- or semicolon-separated values; normalize to commas first - text = str(value).replace(';', ',') - return list_split_no_attrs(text, split_with=',') - + text = str(v).replace(';', ',') + return list_split_no_attrs(text, split_with=',') + return [] +# ====================================================================================== +# Triplet maps +# - DATA and TOPICS: as before +# - NER: separate map that is pushed in a third pass +# ====================================================================================== def default_topic_triplet_map(): - topics_triplet_map_keywords = { - 'ENTITIES':[ - {ET:TOPIC_TYPE, MAKE_ID_UNIQUE:True, FROM_COL: 'Graph_Name', - ATTR_COL:[ - {FROM_COL: 'label', ATTR_NAME:'label', }, - {FROM_COL: 'Graph_Name', ATTR_NAME:'Graph_Name', } - ] - }, - {ET:KEYWORD_TYPE, MAKE_ID_UNIQUE:True}, - ], - 'RELATIONS':[ - {HT:TOPIC_TYPE, R:'child_of', TT:TOPIC_TYPE, EXTRACT_T: get_parent_topic, }, - {HT:TOPIC_TYPE, R:'mentions', TT:KEYWORD_TYPE, EXTRACT_T: get_topic_keywords, }, + return { + 'ENTITIES': [ + {ET: TOPIC_TYPE, MAKE_ID_UNIQUE: True, FROM_COL: 'Graph_Name', + ATTR_COL: [ + {FROM_COL: 'label', ATTR_NAME: 'label'}, + {FROM_COL: 'Graph_Name', ATTR_NAME: 'Graph_Name'}, + ]}, + {ET: KEYWORD_TYPE, MAKE_ID_UNIQUE: True}, + ], + 'RELATIONS': [ + {HT: TOPIC_TYPE, R: 'child_of', TT: TOPIC_TYPE, EXTRACT_T: get_parent_topic}, + {HT: TOPIC_TYPE, R: 'mentions', TT: KEYWORD_TYPE, EXTRACT_T: get_topic_keywords}, ] } - return topics_triplet_map_keywords def default_data_triplet_map(): - data_triplet_map_keywords = { - 'ENTITIES':[ - {ET:TOPIC_TYPE, MAKE_ID_UNIQUE:True, FROM_COL: 'Graph_Name'}, - {ET:DOCUMENT_TYPE, FROM_COL: "doi", - ATTR_COL:[ - {FROM_COL: 'title', ATTR_NAME:'Title', }, - {FROM_COL: 'eid', ATTR_NAME:'EID', }, - {FROM_COL: 's2id', ATTR_NAME:'S2ID', }, - {FROM_COL: 'doi', ATTR_NAME:'DOI', }, - ], - MAKE_ID_UNIQUE:True - }, - {ET:AFFILIATION_IDENTIFIER_TYPE, MAKE_ID_UNIQUE:True}, - {ET:COUNTRY_TYPE, MAKE_ID_UNIQUE:True}, - # {ET:DOCUMENT_TYPE_SCOPUS, MAKE_ID_UNIQUE:True}, - {ET:CATEGORY, MAKE_ID_UNIQUE:True}, - {ET:ACRONYM, MAKE_ID_UNIQUE:True}, - {ET:YEAR_TYPE, FROM_COL: 'year', ATTR_COL:None, ATTR_FUNC:None, MAKE_ID_UNIQUE:True}, - {ET:AUTHOR_ID_TYPE, FROM_COL: 'author_ids', - ATTR_COL:[{FROM_COL: 'authors', ATTR_NAME:'Author_Name', RETREIVAL: split_string, ARGS: None}], - ATTR_FUNC:split_string, ARGS: None, MAKE_ID_UNIQUE:True - }, - {ET:PUBLISHER, FROM_COL: 'publication_name', MAKE_ID_UNIQUE:True}, + return { + 'ENTITIES': [ + {ET: TOPIC_TYPE, MAKE_ID_UNIQUE: True, FROM_COL: 'Graph_Name'}, + {ET: DOCUMENT_TYPE, FROM_COL: "doi", + ATTR_COL: [ + {FROM_COL: 'title', ATTR_NAME: 'Title'}, + {FROM_COL: 'eid', ATTR_NAME: 'EID'}, + {FROM_COL: 's2id', ATTR_NAME: 'S2ID'}, + {FROM_COL: 'doi', ATTR_NAME: 'DOI'}, + ], + MAKE_ID_UNIQUE: True}, + {ET: AFFILIATION_IDENTIFIER_TYPE, MAKE_ID_UNIQUE: True}, + {ET: COUNTRY_TYPE, MAKE_ID_UNIQUE: True}, + {ET: CATEGORY, MAKE_ID_UNIQUE: True}, + {ET: ACRONYM, MAKE_ID_UNIQUE: True}, + {ET: YEAR_TYPE, FROM_COL: 'year', ATTR_COL: None, ATTR_FUNC: None, MAKE_ID_UNIQUE: True}, + {ET: AUTHOR_ID_TYPE, FROM_COL: 'author_ids', + ATTR_COL: [{FROM_COL: 'authors', ATTR_NAME: 'Author_Name', RETREIVAL: split_string, ARGS: None}], + ATTR_FUNC: split_string, ARGS: None, MAKE_ID_UNIQUE: True}, + {ET: PUBLISHER, FROM_COL: 'publication_name', MAKE_ID_UNIQUE: True}, + ], + 'RELATIONS': [ + {HT: DOCUMENT_TYPE, R: 'part_of_topic', TT: TOPIC_TYPE}, + {HT: DOCUMENT_TYPE, R: DOCUMENT_YEAR_RELATION, TT: YEAR_TYPE}, + {HT: AUTHOR_ID_TYPE, R: AUTHOR_DOCUMENT_RELATION, TT: DOCUMENT_TYPE, EXTRACT_H: get_authors_ID}, + {HT: DOCUMENT_TYPE, R: DOCUMENT_AFFILITATION_RELATION, TT: AFFILIATION_IDENTIFIER_TYPE, EXTRACT_T: get_affiliations}, + {HT: AFFILIATION_IDENTIFIER_TYPE, R: AFFILIATION_COUNTRY_RELATION, TT: COUNTRY_TYPE, + EXTRACT_H: get_affiliations, EXTRACT_T: get_countries, PAIRING: INDEX_PAIRING}, + {HT: DOCUMENT_TYPE, R: DOCUMENT_PUBLISHER_RELATION, TT: PUBLISHER}, + {HT: DOCUMENT_TYPE, R: DOCUMENT_CATEGORY_RELATION, TT: CATEGORY, EXTRACT_T: get_categories, PAIRING: HEAD_TO_MANY}, + {HT: DOCUMENT_TYPE, R: DOCUMENT_ACRONYM_RELATION, TT: ACRONYM, EXTRACT_T: get_acronyms}, ], - 'RELATIONS':[ - {HT:DOCUMENT_TYPE, R:'part_of_topic', TT:TOPIC_TYPE}, - {HT:DOCUMENT_TYPE, R:DOCUMENT_YEAR_RELATION, TT:YEAR_TYPE}, - # {HT:DOCUMENT_TYPE, R:DOCUMENT_TOPIC_RELATION, TT:TOPIC_TYPE, EXTRACT_T: get_absolute_cluster,}, - {HT:AUTHOR_ID_TYPE, R:AUTHOR_DOCUMENT_RELATION, TT:DOCUMENT_TYPE, EXTRACT_H: get_authors_ID}, - {HT:DOCUMENT_TYPE, R:DOCUMENT_AFFILITATION_RELATION, TT:AFFILIATION_IDENTIFIER_TYPE, EXTRACT_T: get_affiliations}, - {HT:AFFILIATION_IDENTIFIER_TYPE, R:AFFILIATION_COUNTRY_RELATION, TT:COUNTRY_TYPE, EXTRACT_H: get_affiliations, EXTRACT_T: get_countries, PAIRING: INDEX_PAIRING}, - # {HT:DOCUMENT_TYPE, R:DOCUMENT_CITES_RELATION, TT:DOCUMENT_TYPE_SCOPUS, EXTRACT_T: get_cites, PAIRING: HEAD_TO_MANY}, - # {HT:DOCUMENT_TYPE, R:DOCUMENT_CITED_RELATION, TT:DOCUMENT_TYPE_SCOPUS, EXTRACT_T: get_cited, PAIRING: HEAD_TO_MANY}, - {HT:DOCUMENT_TYPE, R:DOCUMENT_PUBLISHER_RELATION, TT:PUBLISHER}, - {HT:DOCUMENT_TYPE, R:DOCUMENT_CATEGORY_RELATION, TT:CATEGORY, EXTRACT_T: get_categories, PAIRING: HEAD_TO_MANY}, - {HT:DOCUMENT_TYPE, R:DOCUMENT_ACRONYM_RELATION, TT:ACRONYM, EXTRACT_T: get_acronyms}, - ] } - return data_triplet_map_keywords -# ---------- block with defaults ---------- +# --------------------------- NER +import json, ast +from copy import deepcopy + +def _safe_get(row, key, default=None): + try: + if key in row: + return row[key] + except Exception: + pass + if hasattr(row, key): + return getattr(row, key) + if hasattr(row, "get"): + try: + return row.get(key, default) + except Exception: + pass + return default + +def _parse_ner_cell(raw): + if raw is None: + return {} + if isinstance(raw, dict): + return raw + s = str(raw).strip() + if not s or s.lower() == "nan": + return {} + try: + return json.loads(s) + except Exception: + try: + v = ast.literal_eval(s) + return v if isinstance(v, dict) else {} + except Exception: + return {} + +def make_ner_extractor(label: str, + preferred_cols=None, + min_len: int = 1, + dedupe: bool = True): + """ + Returns an extractor function bound to `label`. + Termite will call this with a single dict: {'data': , ...} + """ + preferred_cols = list(preferred_cols or []) + + def _extract(args, _label=label): + row = args.get("data") + print(_label, "row::", row) + if row is None: + return [] + + # discover available keys on the row + try: + keys = list(getattr(row, "index", [])) or list(getattr(row, "keys", lambda: [])()) + except Exception: + keys = [] + + # columns to scan: preferred first, then any *_ents_by_label or ner_by_label + scan_cols = [] + for c in preferred_cols: + if c in keys: + scan_cols.append(c) + for k in keys: + ks = str(k) + if ks == "ner_by_label" or ks.endswith("_ents_by_label") or "ents_by_label" in ks: + if k not in scan_cols: + scan_cols.append(k) + if not scan_cols: + return [] + + seen = set() + out = [] + for c in scan_cols: + d = _parse_ner_cell(_safe_get(row, c, None)) + if not isinstance(d, dict): + continue + items = d.get(_label, []) or [] + for text in items: + t = ("" if text is None else str(text)).strip() + if len(t) < min_len: + continue + if dedupe and t in seen: + continue + seen.add(t) + ent = deepcopy(RETURN_TYPE) + ent[ENTITY] = t + ent[ATTRIBUTES] = [("label", _label), ("source_col", str(c))] + out.append(ent) + return out + + return _extract +def default_ner_triplet_map( + labels=NER_LABELS, + relation_name="mentions", + # prefer reading this column first if present; we still auto-detect *_ents_by_label + ner_col="ner_by_label", + # choose a reliable document ID column; if doi can be missing, create a unified 'doc_id' upstream + document_id_col="doi", +): + m = {"ENTITIES": [], "RELATIONS": []} + + # HEAD: Document entity present in this pass (critical to avoid KeyError/empty triples) + m["ENTITIES"].append({ + ET: DOCUMENT_TYPE, + FROM_COL: document_id_col, + MAKE_ID_UNIQUE: True, + }) + + # TAIL: One ET per label + for lab in labels: + m["ENTITIES"].append({ET: f"NER_{lab}", MAKE_ID_UNIQUE: True}) + + # RELATIONS: bind a label-specific extractor via closure (no ARGS dependency) + for lab in labels: + m["RELATIONS"].append( + { + HT: DOCUMENT_TYPE, + R: relation_name, + TT: f"NER_{lab}", + EXTRACT_T: make_ner_extractor(lab, preferred_cols=[ner_col], min_len=1, dedupe=True), + PAIRING: HEAD_TO_MANY, + } + ) + return m + +# def get_ner_by_label(args): +# """ +# Pull items for ONE spaCy label from a simple column containing a stringified dict: +# {