From 7073144ef3f95b670124d2bb209ca1c9a64875c3 Mon Sep 17 00:00:00 2001 From: schenglee Date: Sun, 13 Nov 2022 01:25:34 +0800 Subject: [PATCH 1/4] amr and rgcn for sp --- .gitignore | 3 + .../amr_graph_construction/__init__.py | 5 + .../amr_graph_construction.py | 410 ++++++ .../pytorch/amr_graph_construction/readme.md | 23 + .../test_amr_construction.py | 19 + .../test_amr_embedding.py | 170 +++ .../mawps/config_for_amr/__init__.py | 0 .../dynamic_amr_undirected.json | 19 + .../dynamic_dependency_undirected.json | 15 + ...emantic_parsing_with_tree_decoder_amr.yaml | 130 ++ ..._parsing_with_tree_decoder_dependency.yaml | 130 ++ .../mawps/src_for_amr/inference.py | 106 ++ .../mawps/src_for_amr/runner.py | 298 +++++ .../mawps/src_for_amr/utils.py | 1125 +++++++++++++++++ examples/pytorch/rgcn/__init__.py | 0 examples/pytorch/rgcn/main.py | 188 +++ examples/pytorch/rgcn/rgcn.py | 116 +- examples/pytorch/rgcn_hetero/__init__.py | 0 examples/pytorch/rgcn_hetero/main.py | 132 ++ examples/pytorch/rgcn_hetero/rgcn_hetero.py | 221 ++++ .../jobs/config_for_amr/__init__.py | 0 .../dynamic_amr_undirected.json | 14 + .../dynamic_dependency_undirected.json | 10 + ...emantic_parsing_with_tree_decoder_amr.yaml | 130 ++ ..._parsing_with_tree_decoder_dependency.yaml | 130 ++ .../graph2tree/jobs/src_for_amr/inference.py | 106 ++ .../graph2tree/jobs/src_for_amr/runner.py | 286 +++++ .../graph2tree/jobs/src_for_amr/utils.py | 1125 +++++++++++++++++ graph4nlp/pytorch/data/data.py | 48 +- graph4nlp/pytorch/data/dataset.py | 56 +- graph4nlp/pytorch/datasets/jobs.py | 6 +- graph4nlp/pytorch/datasets/mawps.py | 6 +- graph4nlp/pytorch/modules/utils/tree_utils.py | 3 +- .../pytorch/modules/utils/vocab_utils.py | 17 + .../test/data_structure/test_graphdata.py | 15 +- .../test_embedding_construction.py | 42 +- 36 files changed, 5014 insertions(+), 90 deletions(-) create mode 100644 examples/pytorch/amr_graph_construction/__init__.py create mode 100644 examples/pytorch/amr_graph_construction/amr_graph_construction.py create mode 100644 examples/pytorch/amr_graph_construction/readme.md create mode 100644 examples/pytorch/amr_graph_construction/test_amr_construction.py create mode 100644 examples/pytorch/amr_graph_construction/test_amr_embedding.py create mode 100644 examples/pytorch/math_word_problem/mawps/config_for_amr/__init__.py create mode 100644 examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_amr_undirected.json create mode 100644 examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_dependency_undirected.json create mode 100644 examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml create mode 100644 examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml create mode 100644 examples/pytorch/math_word_problem/mawps/src_for_amr/inference.py create mode 100644 examples/pytorch/math_word_problem/mawps/src_for_amr/runner.py create mode 100644 examples/pytorch/math_word_problem/mawps/src_for_amr/utils.py create mode 100644 examples/pytorch/rgcn/__init__.py create mode 100644 examples/pytorch/rgcn/main.py create mode 100644 examples/pytorch/rgcn_hetero/__init__.py create mode 100644 examples/pytorch/rgcn_hetero/main.py create mode 100644 examples/pytorch/rgcn_hetero/rgcn_hetero.py create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/__init__.py create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_amr_undirected.json create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py create mode 100644 examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/utils.py diff --git a/.gitignore b/.gitignore index 1dd7612f..cab75c64 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,6 @@ cscope.* # config file /config local_scripts/ + +**/amr_graph_construction/mawps/* + diff --git a/examples/pytorch/amr_graph_construction/__init__.py b/examples/pytorch/amr_graph_construction/__init__.py new file mode 100644 index 00000000..a74e1743 --- /dev/null +++ b/examples/pytorch/amr_graph_construction/__init__.py @@ -0,0 +1,5 @@ +from .amr_graph_construction import AMRGraphConstruction + +__all__ = [ + "AMRGraphConstruction", +] \ No newline at end of file diff --git a/examples/pytorch/amr_graph_construction/amr_graph_construction.py b/examples/pytorch/amr_graph_construction/amr_graph_construction.py new file mode 100644 index 00000000..ddb31ad8 --- /dev/null +++ b/examples/pytorch/amr_graph_construction/amr_graph_construction.py @@ -0,0 +1,410 @@ +import copy +import json +import spacy +import amrlib +from collections import defaultdict +from amrlib.alignments.faa_aligner import FAA_Aligner +from graph4nlp.pytorch.data.data import GraphData +from graph4nlp.pytorch.modules.graph_construction.base import StaticGraphConstructionBase + + +class AMRGraphConstruction(StaticGraphConstructionBase): + """ + Dependency-parsing-tree based graph construction class + + Parameters + ---------- + vocab: VocabModel + Vocabulary including all words appeared in graphs. + """ + + def __init__( + self, + vocab, + ): + super(AMRGraphConstruction, self).__init__() + self.vocab = vocab + self.verbose = 1 + + def add_vocab(self, g): + """ + Add node tokens appeared in graph g to vocabulary. + + Parameters + ---------- + g: GraphData + Graph data-structure. + + """ + for i in range(g.get_node_num()): + attr = g.get_node_attrs(i)[i] + self.vocab.word_vocab._add_words([attr["token"]]) + + @classmethod + def parsing(cls, raw_text_data, nlp_processor, processor_args): + """ + + Parameters + ---------- + raw_text_data: str + + Returns + ------- + parsed_results: list[dict] + Each sentence is a dict. All sentences are packed by a list. + key, value + "node_num": int + the node amount + "node_content": list[dict] + The list consisting node information. Each node is organized by a dict. + 'token': str + the entire word + 'id': int + the node token's id which will be used in GraphData + 'variable': str + the variable + 'type': int + the node type + 'sentence_id': int + the sentence id of the node + "graph_content": list[dict] + The list consisting edge information. Each edge is organized by a dict. + "edge_type": str + The edge type token, eg: 'ARG1' + 'src': int + The source node ``id`` + 'tgt': int. + The target node ``id`` + 'sentence_id': int + The sentence id of the edge + "sentence": str + The original sentence of the amr graph. + "mapping": dict[list] + The mapping between sequence token index and node or edge index. + """ + amrlib.setup_spacy_extension() + nlp = spacy.load('en_core_web_sm') + doc = nlp(raw_text_data) + parsed_results = [] + graphs = doc._.to_amr() + st = [] + for ind, (graph, sentences) in enumerate(zip(graphs, doc.sents)): + node_item = [] + parsed_sent = [] + st = [] + node_id = 0 + node2id = {} + + index = {} + size_son = {} + + for line in graph.splitlines(): + if line[0] == '#': + continue + l = line.strip().split() + # add new node + if line.find('/') != -1 and '/' in l: + variable = l[l.index('/') - 1].strip('(') + concept = l[l.index('/') + 1].strip(')') + if '-' in concept: + concept = concept.split('-')[0] + assert concept is not '' + node = { + "variable": variable, + "id": node_id, + "token": concept, + "type": 4, # 4 for amr graph node + "sentence_id": len(parsed_results), + } + node2id[variable] = node_id + nodeid_now = node_id + node_item.append(node) + node_id += 1 + else: + variable = l[-1].strip(')').strip('"') + if variable is '': + variable = l[1].strip(')').strip('"') + if variable is '': + continue + if variable not in node2id: + node = { + "variable": None, + "id": node_id, + "token": variable, + "type": 4, # 4 for amr graph node + "sentence_id": len(parsed_results), + } + nodeid_now = node_id + node_item.append(node) + node_id += 1 + else: + nodeid_now = node2id[variable] + cnt = 0 + for c in line: + if c != ' ': + break + cnt += 1 + while len(st) and st[-1][0] >= cnt: + st.pop() + nodeid_now = int(nodeid_now) + # add new edge + if line.find(':') != -1: + fa = st[-1][1] + pos = st[-1][2] + dep_info = { + "src": fa, + "tgt": nodeid_now, + "edge_type": l[0][1:], + "sentence_id": len(parsed_results), + } + parsed_sent.append(dep_info) + pos_now = pos + '.' + str(size_son[pos] + 1) + size_son[pos_now] = 0 + index[pos_now] = nodeid_now + index[pos_now + '.r'] = len(parsed_sent) - 1 + size_son[pos] += 1 + else: + pos_now = '1' + index[pos_now] = nodeid_now + size_son[pos_now] = 0 + + # push the node to stack + st.append((cnt, nodeid_now, pos_now)) + + inference = FAA_Aligner() + if graph is None or sentences.text is '' or len(graph) == 0 or len(sentences) == 0: + continue + try: + _, alignment_strings = inference.align_sents([sentences.text], [graph]) + except Exception as e: + continue + alignment = alignment_strings[0].strip().split(' ') + sentences_token = sentences.text.strip().split(' ') + mapping = defaultdict(list) + for relation in alignment: + assert('-' in relation) + src = relation.split('-')[0] + tgt = relation.split('-')[1] + assert(tgt in index) + assert(int(src) <= len(sentences_token)) + if 'r' in tgt: + mapping[index[tgt]].append((int(src), "edge")) + else: + mapping[index[tgt]].append((int(src), "node")) + + dep_dict = nlp_processor.annotate(sentences.text, properties=processor_args) + pos_tag = [tokens["pos"] for tokens in dep_dict["sentences"][0]["tokens"]] + entity_label = [tokens["ner"] for tokens in dep_dict["sentences"][0]["tokens"]] + parsed_results.append( + {"graph_content": parsed_sent, "node_content": node_item, "node_num": node_id, + "sentence": sentences.text, "mapping": mapping, "pos_tag": pos_tag, "entity_label": entity_label} + ) + + return parsed_results + + @classmethod + def static_topology( + cls, + raw_text_data, + merge_strategy=None, + edge_strategy=None, + nlp_processor=None, + processor_args=None, + verbose=0, + ): + """ + Graph building method. + + Parameters + ---------- + raw_text_data: str or list[list] + Raw text data, it can be multi-sentences. + When it is ``str`` type, it is the raw text. + When it is ``list[list]`` type, it is the tokenized token lists. + verbose: int, default=0 + Whether to output log infors. Set 1 to output more infos. + Returns + ------- + joint_graph: GraphData + The merged graph data-structure. + """ + cls.verbose = verbose + parsed_results = cls.parsing(raw_text_data, nlp_processor, processor_args) + + sub_graphs = [] + for parsed_sent in parsed_results: + graph = cls._construct_static_graph(parsed_sent) + sub_graphs.append(graph) + joint_graph = cls._graph_connect(sub_graphs) + return joint_graph + + @classmethod + def _construct_static_graph(cls, parsed_object): + """ + Build dependency-parsing-tree based graph for single sentence. + + Parameters + ---------- + parsed_object: dict + The parsing tree. + + Returns + ------- + graph: GraphData + graph structure for single sentence + """ + ret_graph = GraphData() + node_num = parsed_object["node_num"] + assert node_num > 0 + ret_graph.add_nodes(node_num) + head_node = 0 + tail_node = node_num - 1 + + # insert node attributes + node_objects = parsed_object["node_content"] + for node in node_objects: + ret_graph.node_attributes[node["id"]]["type"] = node["type"] + ret_graph.node_attributes[node["id"]]["variable"] = node["variable"] + ret_graph.node_attributes[node["id"]]["token"] = node["token"] + ret_graph.node_attributes[node["id"]]["sentence_id"] = node["sentence_id"] + ret_graph.node_attributes[node["id"]]["id"] = node["id"] + ret_graph.node_attributes[node["id"]]["head"] = False + ret_graph.node_attributes[node["id"]]["tail"] = False + + for dep_info in parsed_object["graph_content"]: + ret_graph.add_edge(dep_info["src"], dep_info["tgt"]) + edge_idx = ret_graph.edge_ids(dep_info["src"], dep_info["tgt"])[0] + ret_graph.edge_attributes[edge_idx]["token"] = dep_info["edge_type"] + ret_graph.edge_attributes[edge_idx]["sentence_id"] = dep_info["sentence_id"] + + ret_graph.node_attributes[head_node]["head"] = True + ret_graph.node_attributes[tail_node]["tail"] = True + + # add graph attributes + ret_graph.graph_attributes["mapping"] = parsed_object["mapping"] + ret_graph.graph_attributes["sentence"] = parsed_object["sentence"] + ret_graph.graph_attributes["pos_tag"] = parsed_object["pos_tag"] + ret_graph.graph_attributes["entity_label"] = parsed_object["entity_label"] + + return ret_graph + + @classmethod + def _graph_connect(cls, nx_graph_list, merge_strategy=None): + """ + This method will merge the sub-graphs into one graph. + + Parameters + ---------- + nx_graph_list: list[GraphData] + The list of all sub-graphs. + + Returns + ------- + joint_graph: GraphData + The merged graph structure. + """ + if cls.verbose > 0: + print("sub_graph print") + for i, s_g in enumerate(nx_graph_list): + print("-------------------------") + print("sub-graph: {}".format(i)) + print("node_num: {}".format(s_g.get_node_num())) + for i in range(s_g.get_node_num()): + print(s_g.get_node_attrs(i)) + print("edge_num: {}".format(s_g.get_edge_num())) + print(s_g.get_all_edges()) + for i in range(s_g.get_edge_num()): + print(i, s_g.edge_attributes[i]) + print("sentence: {}".format(s_g.graph_attributes["sentence"])) + print("mapping: {}".format(s_g.graph_attributes["mapping"])) + print("*************************************") + if len(nx_graph_list) == 0: + raise RuntimeError("There is no graph needed to merge.") + node_num_list = [s_g.get_node_num() for s_g in nx_graph_list] + node_num = sum(node_num_list) + g = GraphData() + g.add_nodes(node_num) + node_idx_off = 0 + # copy graph attributes + g.graph_attributes["mapping"] = list() + g.graph_attributes["sentence"] = list() + g.graph_attributes["pos_tag"] = list() + g.graph_attributes["entity_label"] = list() + for graph in nx_graph_list: + g.graph_attributes["mapping"].append(graph.graph_attributes["mapping"]) + g.graph_attributes["sentence"].append(graph.graph_attributes["sentence"]) + g.graph_attributes["pos_tag"].append(graph.graph_attributes["pos_tag"]) + g.graph_attributes["entity_label"].append(graph.graph_attributes["entity_label"]) + + # copy edges + for s_g in nx_graph_list: + for edge in s_g.get_all_edges(): + src, tgt = edge + edge_idx_old = s_g.edge_ids(src, tgt)[0] + g.add_edge(src + node_idx_off, tgt + node_idx_off) + edge_idx_new = g.edge_ids(src + node_idx_off, tgt + node_idx_off)[0] + if cls.verbose > 0: + print(edge_idx_new, edge_idx_old) + print(s_g.edge_attributes[edge_idx_old], "--------") + g.edge_attributes[edge_idx_new] = copy.deepcopy(s_g.edge_attributes[edge_idx_old]) + + for key, value in enumerate(s_g.node_attributes): + g.node_attributes[key + node_idx_off] = copy.deepcopy(value) + node_idx_off += s_g.get_node_num() + + headtail_list = [] + + head = -1 + tail = -1 + + node_idx_off = 0 + for i in range(len(nx_graph_list)): + for node_idx, node_attrs in enumerate(nx_graph_list[i].node_attributes): + if node_attrs["head"] is True: + head = node_idx + node_idx_off + if node_attrs["tail"] is True: + tail = node_idx + node_idx_off + assert head != -1 + assert tail != -1 + headtail_list.append((head, tail)) + head = -1 + tail = -1 + node_idx_off += node_num_list[i] + + head_g = headtail_list[0][0] + tail_g = headtail_list[-1][1] + + if merge_strategy is None or merge_strategy == "tailhead": + + src_list = [] + tgt_list = [] + + for i in range(len(headtail_list) - 1): + src_list.append(headtail_list[i][1]) + tgt_list.append(headtail_list[i + 1][0]) + if cls.verbose > 0: + print("merged edges") + print("src list:", src_list) + print("tgt list:", tgt_list) + g.add_edges(src_list, tgt_list) + else: + raise NotImplementedError() + + for node_idx, node_attrs in enumerate(g.node_attributes): + node_attrs["head"] = node_idx == head_g + node_attrs["tail"] = node_idx == tail_g + + if cls.verbose > 0: + print("-----------------------------") + print("merged graph") + print("node_num: {}".format(g.get_node_num())) + for i in range(g.get_node_num()): + print(g.get_node_attrs(i)) + print("edge_num: {}".format(g.get_edge_num())) + print(g.get_all_edges()) + for i in range(g.get_edge_num()): + print(i, g.edge_attributes[i]) + return g + + def forward(self, batch_graphdata: list): + raise RuntimeError("This interface is removed.") diff --git a/examples/pytorch/amr_graph_construction/readme.md b/examples/pytorch/amr_graph_construction/readme.md new file mode 100644 index 00000000..d6fe6b3e --- /dev/null +++ b/examples/pytorch/amr_graph_construction/readme.md @@ -0,0 +1,23 @@ +# Graph2Tree for math word problem (MWP) For AMR graph and RGCN + +## Setup + +### Install and set + +install the amrlib and the fast_align, and set the environment variables as follows +```bash +export FABIN_DIR=path_to_fast_align +export TOKENIZERS_PARALLELISM=True +``` + +### Run with following + +#### Run with amr graph adn RGCN +```python +python examples/pytorch/math_word_problem/mawps/src_for_amr/runner.py -json examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_amr_undirected.json +``` +#### Run with dependency graph and RGCN +```python +python examples/pytorch/math_word_problem/mawps/src_for_amr/runner.py -json examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_dependency_undirected.json +``` + diff --git a/examples/pytorch/amr_graph_construction/test_amr_construction.py b/examples/pytorch/amr_graph_construction/test_amr_construction.py new file mode 100644 index 00000000..210665af --- /dev/null +++ b/examples/pytorch/amr_graph_construction/test_amr_construction.py @@ -0,0 +1,19 @@ +from amr_graph_construction import ( + AMRGraphConstruction, +) + + +def test_amr(): + raw_data = ( + "find all languageid0 job in locid0" + ) + + AMRGraphConstruction.static_topology( + raw_data, + verbose=1, + ) + pass + + +if __name__ == "__main__": + test_amr() diff --git a/examples/pytorch/amr_graph_construction/test_amr_embedding.py b/examples/pytorch/amr_graph_construction/test_amr_embedding.py new file mode 100644 index 00000000..c4778f2b --- /dev/null +++ b/examples/pytorch/amr_graph_construction/test_amr_embedding.py @@ -0,0 +1,170 @@ +from graph4nlp.pytorch.datasets.jobs import JobsDataset +from graph4nlp.pytorch.data.dataset import Text2TextDataset +from copy import deepcopy +import numpy as np +import torch.utils.data + +from ..data.data import GraphData + +from ..modules.utils.padding_utils import pad_2d_vals_no_size +from ..modules.utils.tree_utils import Vocab as VocabForTree + +class AMRDataset(Text2TextDataset): + """ + + Parameters + ---------- + root_dir: str + The path of dataset. + graph_construction_name: str + The name of graph construction method. E.g., "dependency". + Note that if it is in the provided graph names (i.e., "dependency", \ + "constituency", "ie", "node_emb", "node_emb_refine"), the following \ + parameters are set by default and users can't modify them: + 1. ``topology_builder`` + 2. ``static_or_dynamic`` + If you need to customize your graph construction method, you should rename the \ + ``graph_construction_name`` and set the parameters above. + topology_builder: GraphConstructionBase, default=None + The graph construction class. + topology_subdir: str + The directory name of processed path. + static_or_dynamic: str, default='static' + The graph type. Expected in ('static', 'dynamic') + edge_strategy: str, default=None + The edge strategy. Expected in (None, 'homogeneous', 'as_node'). + If set `None`, it will be 'homogeneous'. + merge_strategy: str, default=None + The strategy to merge sub-graphs. Expected in (None, 'tailhead', 'user_define'). + If set `None`, it will be 'tailhead'. + share_vocab: bool, default=False + Whether to share the input vocabulary with the output vocabulary. + dynamic_init_graph_name: str, default=None + The graph name of the initial graph. Expected in (None, "line", "dependency", \ + "constituency"). + Note that if it is in the provided graph names (i.e., "line", "dependency", \ + "constituency"), the following parameters are set by default and users \ + can't modify them: + 1. ``dynamic_init_topology_builder`` + If you need to customize your graph construction method, you should rename the \ + ``graph_name`` and set the parameters above. + dynamic_init_topology_builder: GraphConstructionBase + The graph construction class. + dynamic_init_topology_aux_args: None, + TBD. + """ + + @property + def raw_file_names(self): + """3 reserved keys: 'train', 'val' (optional), 'test'. Represent the split of dataset.""" + return {"train": "train.txt", "test": "test.txt"} + + @property + def processed_file_names(self): + """At least 3 reserved keys should be fiiled: 'vocab', 'data' and 'split_ids'.""" + return {"vocab": "vocab.pt", "data": "data.pt"} + + def download(self): + # raise NotImplementedError( + # 'This dataset is now under test and cannot be downloaded. + # Please prepare the raw data yourself.') + return + + def __init__( + self, + root_dir, + topology_subdir, + graph_construction_name, + static_or_dynamic="static", + topology_builder=None, + merge_strategy="tailhead", + edge_strategy=None, + dynamic_init_graph_name=None, + dynamic_init_topology_builder=None, + dynamic_init_topology_aux_args=None, + pretrained_word_emb_name="6B", + pretrained_word_emb_url=None, + pretrained_word_emb_cache_dir=None, + seed=None, + word_emb_size=300, + share_vocab=True, + lower_case=True, + thread_number=1, + port=9000, + for_inference=None, + reused_vocab_model=None, + ): + # Initialize the dataset. If the preprocessed files are not found, + # then do the preprocessing and save them. + super(JobsDataset, self).__init__( + root_dir=root_dir, + topology_builder=topology_builder, + topology_subdir=topology_subdir, + graph_construction_name=graph_construction_name, + static_or_dynamic=static_or_dynamic, + edge_strategy=edge_strategy, + merge_strategy=merge_strategy, + share_vocab=share_vocab, + lower_case=lower_case, + pretrained_word_emb_name=pretrained_word_emb_name, + pretrained_word_emb_url=pretrained_word_emb_url, + pretrained_word_emb_cache_dir=pretrained_word_emb_cache_dir, + seed=seed, + word_emb_size=word_emb_size, + thread_number=thread_number, + port=port, + dynamic_init_graph_name=dynamic_init_graph_name, + dynamic_init_topology_builder=dynamic_init_topology_builder, + dynamic_init_topology_aux_args=dynamic_init_topology_aux_args, + for_inference=for_inference, + reused_vocab_model=reused_vocab_model, + ) + @classmethod + def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): + + item = deepcopy(data_item) + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = vocab_model.in_word_vocab.getIndex(node_token, use_ie) + graph.node_attributes[node_idx]["token_id"] = node_token_id + + token_matrix.append([node_token_id]) + if use_ie: + for i in range(len(token_matrix)): + token_matrix[i] = np.array(token_matrix[i][0]) + token_matrix = pad_2d_vals_no_size(token_matrix) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + else: + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + + if use_ie and "token" in graph.edge_attributes[0].keys(): + edge_token_matrix = [] + for edge_idx in range(graph.get_edge_num()): + edge_token = graph.edge_attributes[edge_idx]["token"] + edge_token_id = vocab_model.in_word_vocab.getIndex(edge_token, use_ie) + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + edge_token_matrix.append([edge_token_id]) + if use_ie: + for i in range(len(edge_token_matrix)): + edge_token_matrix[i] = np.array(edge_token_matrix[i][0]) + edge_token_matrix = pad_2d_vals_no_size(edge_token_matrix) + edge_token_matrix = torch.tensor(edge_token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = edge_token_matrix + + tgt = item.output_text + if isinstance(tgt, str): + tgt_token_id = vocab_model.out_word_vocab.to_index_sequence(tgt) + tgt_token_id.append(vocab_model.out_word_vocab.EOS) + tgt_token_id = np.array(tgt_token_id) + item.output_np = tgt_token_id + return item + + def vectorization(self, data_items): + for idx in range(len(data_items)): + data_items[idx] = self._vectorize_one_dataitem( + data_items[idx], self.vocab_model + ) diff --git a/examples/pytorch/math_word_problem/mawps/config_for_amr/__init__.py b/examples/pytorch/math_word_problem/mawps/config_for_amr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_amr_undirected.json b/examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_amr_undirected.json new file mode 100644 index 00000000..6c83b788 --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_amr_undirected.json @@ -0,0 +1,19 @@ +{ + "config_path": "examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml", + "checkpoint_args.checkpoint_name": "node_emb_sage_undirected.pt", + "checkpoint_args.out_dir": "examples/pytorch/math_word_problem/mawps/config_for_amr/save", + "env_args.gpuid": 0, + "training_args.batch_size": 20, + "training_args.max_epochs": 150, + "model_args.graph_construction_args.graph_construction_share.root_dir": "examples/pytorch/math_word_problem/mawps/mawps_data", + "inference_args.inference_data_dir": "examples/pytorch/math_word_problem/mawps/mawps_data_for_inference", + "model_args.decoder_args.rnn_decoder_share.dropout": 0.3, + "model_args.decoder_args.rnn_decoder_private.max_decoder_step": 35, + "model_args.decoder_args.rnn_decoder_private.max_tree_depth": 8, + "preprocessing_args.pretrained_word_emb_name": "6B", + "model_args.graph_construction_name": "amr", + "model_args.graph_construction_args.graph_construction_share.topology_subdir": "AMRGraphForRGCN", + "model_args.graph_initialization_args.embedding_style.single_token_item": false, + "model_args.graph_initialization_args.embedding_style.emb_strategy": "w2v_bilstm_amr_pos" + } + \ No newline at end of file diff --git a/examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_dependency_undirected.json b/examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_dependency_undirected.json new file mode 100644 index 00000000..43af5eb9 --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/config_for_amr/dynamic_dependency_undirected.json @@ -0,0 +1,15 @@ +{ + "config_path": "examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml", + "checkpoint_args.checkpoint_name": "node_emb_sage_undirected.pt", + "checkpoint_args.out_dir": "examples/pytorch/math_word_problem/mawps/config_for_amr/save", + "env_args.gpuid": 0, + "training_args.batch_size": 20, + "training_args.max_epochs": 150, + "model_args.graph_construction_args.graph_construction_share.root_dir": "examples/pytorch/math_word_problem/mawps/mawps_data", + "inference_args.inference_data_dir": "examples/pytorch/math_word_problem/mawps/mawps_data_for_inference", + "model_args.decoder_args.rnn_decoder_share.dropout": 0.3, + "model_args.decoder_args.rnn_decoder_private.max_decoder_step": 35, + "model_args.decoder_args.rnn_decoder_private.max_tree_depth": 8, + "preprocessing_args.pretrained_word_emb_name": "6B" + } + \ No newline at end of file diff --git a/examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml b/examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml new file mode 100644 index 00000000..867a50f3 --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml @@ -0,0 +1,130 @@ +# Users can import user customized yaml files or library provided default yaml files +includes: [] + # - $/configs/defaults.yaml + # In the above example path, '$/' indicates the library directory which contains the config folder + + +preprocessing_args: + min_freq: 1 + pretrained_word_emb_name: null + + +model_args: + graph_construction_name: "dependency" + graph_initialization_name: "defaults" + graph_embedding_name: "rgcn" + decoder_name: "stdtree" + + + graph_construction_args: + graph_construction_share: + root_dir: 'examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data' + topology_subdir: 'DependencyGraphForRGCN' + thread_number: 15 + share_vocab: True + port: 9000 + timeout: 15000 + + nlp_processor_args: + name: "stanza" + args: + annotators: ["tokenize", "ssplit", "pos", "ner"] + corenlp_dir: "./corenlp" + endpoint: "http://localhost:9002" + memory: "4G" + properties: + tokenize.options: + splitHyphenated: False + normalizeParentheses: False + normalizeOtherBrackets: False + tokenize.whitespace: True + ssplit.isOneSentence: True + + graph_construction_private: + edge_strategy: 'heterogeneous' + merge_strategy: 'tailhead' + sequential_link: true + as_node: false + sim_metric_type: 'weighted_cosine' + num_heads: 1 + top_k_neigh: null + epsilon_neigh: 0.5 + smoothness_ratio: 0.1 + connectivity_ratio: 0.05 + sparsity_ratio: 0.1 + + graph_initialization_args: + input_size: 300 + hidden_size: 300 + word_dropout: 0.1 + rnn_dropout: 0.1 + # word_dropout: 0.2 + # rnn_dropout: 0.3 + fix_bert_emb: false + fix_word_emb: false + embedding_style: + single_token_item: true + emb_strategy: "w2v_bilstm" + num_rnn_layers: 1 + bert_model_name: null + bert_lower_case: null + + graph_embedding_args: + graph_embedding_share: + num_layers: 1 + input_size: 300 + hidden_size: 300 + output_size: 300 + direction_option: "undirected" + feat_drop: 0.0 + attn_drop: 0.0 + graph_embedding_private: + aggregator_type: "lstm" + bias: true + norm: null + activation: "relu" + use_edge_weight: true + + decoder_args: + rnn_decoder_share: + rnn_type: "lstm" + input_size: 300 + hidden_size: 300 + rnn_emb_input_size: 300 + use_copy: true + graph_pooling_strategy: null + attention_type: "uniform" + fuse_strategy: "concatenate" + dropout: 0.1 + teacher_forcing_rate: 1.0 + rnn_decoder_private: + max_decoder_step: 50 + max_tree_depth: 50 + use_sibling: false + +training_args: + learning_rate: 0.001 + init_weight: 0.08 + weight_decay: 0 + max_epochs: 150 + grad_clip: 5 + batch_size: 20 + + +inference_args: + beam_size: 4 + inference_data_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data_inference" + + +evaluation_args: + # Metrics for evaluation + + +checkpoint_args: + out_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/save" + checkpoint_name: "best.pt" + + +env_args: + seed: 0 + gpuid: -1 diff --git a/examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml b/examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml new file mode 100644 index 00000000..52341500 --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml @@ -0,0 +1,130 @@ +# Users can import user customized yaml files or library provided default yaml files +includes: [] + # - $/configs/defaults.yaml + # In the above example path, '$/' indicates the library directory which contains the config folder + + +preprocessing_args: + min_freq: 1 + pretrained_word_emb_name: null + + +model_args: + graph_construction_name: "dependency" + graph_initialization_name: "defaults" + graph_embedding_name: "rgcn" + decoder_name: "stdtree" + + + graph_construction_args: + graph_construction_share: + root_dir: 'examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data' + topology_subdir: 'DependencyGraphForRGCN' + thread_number: 15 + share_vocab: True + port: 9000 + timeout: 15000 + + nlp_processor_args: + name: "stanza" + args: + annotators: ["tokenize", "ssplit", "pos", "depparse"] + corenlp_dir: "./corenlp" + endpoint: "http://localhost:9002" + memory: "4G" + properties: + tokenize.options: + splitHyphenated: False + normalizeParentheses: False + normalizeOtherBrackets: False + tokenize.whitespace: True + ssplit.isOneSentence: False + + graph_construction_private: + edge_strategy: 'heterogeneous' + merge_strategy: 'tailhead' + sequential_link: true + as_node: false + sim_metric_type: 'weighted_cosine' + num_heads: 1 + top_k_neigh: null + epsilon_neigh: 0.5 + smoothness_ratio: 0.1 + connectivity_ratio: 0.05 + sparsity_ratio: 0.1 + + graph_initialization_args: + input_size: 300 + hidden_size: 300 + word_dropout: 0.1 + rnn_dropout: 0.1 + # word_dropout: 0.2 + # rnn_dropout: 0.3 + fix_bert_emb: false + fix_word_emb: false + embedding_style: + single_token_item: true + emb_strategy: "w2v_bilstm" + num_rnn_layers: 1 + bert_model_name: null + bert_lower_case: null + + graph_embedding_args: + graph_embedding_share: + num_layers: 1 + input_size: 300 + hidden_size: 300 + output_size: 300 + direction_option: "undirected" + feat_drop: 0.0 + attn_drop: 0.0 + graph_embedding_private: + aggregator_type: "lstm" + bias: true + norm: null + activation: "relu" + use_edge_weight: true + + decoder_args: + rnn_decoder_share: + rnn_type: "lstm" + input_size: 300 + hidden_size: 300 + rnn_emb_input_size: 300 + use_copy: true + graph_pooling_strategy: null + attention_type: "uniform" + fuse_strategy: "concatenate" + dropout: 0.1 + teacher_forcing_rate: 1.0 + rnn_decoder_private: + max_decoder_step: 50 + max_tree_depth: 50 + use_sibling: false + +training_args: + learning_rate: 0.001 + init_weight: 0.08 + weight_decay: 0 + max_epochs: 150 + grad_clip: 5 + batch_size: 20 + + +inference_args: + beam_size: 4 + inference_data_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data_inference" + + +evaluation_args: + # Metrics for evaluation + + +checkpoint_args: + out_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/save" + checkpoint_name: "best.pt" + + +env_args: + seed: 0 + gpuid: -1 diff --git a/examples/pytorch/math_word_problem/mawps/src_for_amr/inference.py b/examples/pytorch/math_word_problem/mawps/src_for_amr/inference.py new file mode 100644 index 00000000..0bbfd62f --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/src_for_amr/inference.py @@ -0,0 +1,106 @@ +""" + The inference code. + In this file, we will run the inference by using the prediction API \ + in the GeneratorInferenceWrapper. + The GeneratorInferenceWrapper takes the raw inputs and produce the outputs. +""" +import argparse +import random +import warnings +import numpy as np +import torch +from utils import AMRDataItem + +from graph4nlp.pytorch.datasets.mawps import MawpsDatasetForTree, tokenize_mawps +from graph4nlp.pytorch.inference_wrapper.generator_inference_wrapper_for_tree import ( + GeneratorInferenceWrapper, +) +from graph4nlp.pytorch.modules.utils.config_utils import load_json_config +from examples.pytorch.amr_graph_construction.amr_graph_construction import AMRGraphConstruction +from utils import AMRDataItem, RGCNGraph2Tree, InferenceText2TreeDataset +warnings.filterwarnings("ignore") + + +class Mawps: + def __init__(self, opt=None): + super(Mawps, self).__init__() + self.opt = opt + + seed = self.opt["env_args"]["seed"] + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if self.opt["env_args"]["gpuid"] == -1: + self.device = torch.device("cpu") + else: + self.device = torch.device("cuda:{}".format(self.opt["env_args"]["gpuid"])) + + self._build_model() + + def _build_model(self): + self.model = RGCNGraph2Tree.load_checkpoint( + self.opt["checkpoint_args"]["out_dir"], self.opt["checkpoint_args"]["checkpoint_name"] + ).to(self.device) + + self.inference_tool = GeneratorInferenceWrapper( + cfg=self.opt, + model=self.model, + beam_size=2, + lower_case=True, + tokenizer=tokenize_mawps, + dataset=InferenceText2TreeDataset, + data_item=AMRDataItem, + topology_builder=(AMRGraphConstruction if self.model.graph_construction_name == "amr" else None) + ) + + @torch.no_grad() + def translate(self): + self.model.eval() + ret = self.inference_tool.predict( + raw_contents=[ + "2 dogs are barking . 1 more dogs start to bark . how many dogs are barking" + ], + batch_size=1, + ) + print(ret) + + +################################################################################ +# ArgParse and Helper Functions # +################################################################################ +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-json_config", + "--json_config", + required=True, + type=str, + help="path to the json config file", + ) + args = vars(parser.parse_args()) + + return args + + +def print_config(config): + import pprint + + print("**************** MODEL CONFIGURATION ****************") + pprint.pprint(config) + print("**************** MODEL CONFIGURATION ****************") + + +if __name__ == "__main__": + import platform + import multiprocessing + + #if platform.system() == "Darwin": + multiprocessing.set_start_method("spawn") + + cfg = get_args() + config = load_json_config(cfg["json_config"]) + # print_config(config) + + runner = Mawps(opt=config) + runner.translate() diff --git a/examples/pytorch/math_word_problem/mawps/src_for_amr/runner.py b/examples/pytorch/math_word_problem/mawps/src_for_amr/runner.py new file mode 100644 index 00000000..23f79a60 --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/src_for_amr/runner.py @@ -0,0 +1,298 @@ +import argparse +import copy +import random +import time +from typing import Union +import warnings +import numpy as np +import torch +import torch.optim as optim +import torch.nn.functional as F +from copy import deepcopy +from torch.utils.data import DataLoader +from tqdm import tqdm +from graph4nlp.pytorch.data.data import GraphData +from graph4nlp.pytorch.data.dataset import DataItem, Text2TreeDataItem, Text2TreeDataset + +from graph4nlp.pytorch.datasets.mawps import MawpsDatasetForTree +from graph4nlp.pytorch.models.graph2tree import Graph2Tree +from graph4nlp.pytorch.modules.utils.config_utils import load_json_config +from graph4nlp.pytorch.modules.utils.tree_utils import Tree + +from examples.pytorch.rgcn.rgcn import RGCN +from examples.pytorch.amr_graph_construction.amr_graph_construction import AMRGraphConstruction +from utils import AMRDataItem, EdgeText2TreeDataset, RGCNGraph2Tree, AMRGraph2Tree + +warnings.filterwarnings("ignore") + +class Mawps: + def __init__(self, opt=None): + super(Mawps, self).__init__() + self.opt = opt + + seed = self.opt["env_args"]["seed"] + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if self.opt["env_args"]["gpuid"] == -1: + self.device = torch.device("cpu") + else: + self.device = torch.device("cuda:{}".format(self.opt["env_args"]["gpuid"])) + + self.use_copy = self.opt["model_args"]["decoder_args"]["rnn_decoder_share"]["use_copy"] + self.use_share_vocab = self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["share_vocab"] + self.data_dir = self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["root_dir"] + + self._build_dataloader() + self._build_model() + self._build_optimizer() + + def _build_dataloader(self): + graph_type = self.opt["model_args"]["graph_construction_name"] + para_dic = { + "root_dir": self.data_dir, + "word_emb_size": self.opt["model_args"]["graph_initialization_args"]["input_size"], + "topology_subdir": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["topology_subdir"], + "edge_strategy": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_private" + ]["edge_strategy"], + "graph_construction_name": self.opt["model_args"]["graph_construction_name"], + "share_vocab": self.use_share_vocab, + "enc_emb_size": self.opt["model_args"]["graph_initialization_args"]["input_size"], + "dec_emb_size": self.opt["model_args"]["decoder_args"]["rnn_decoder_share"][ + "input_size" + ], + "dynamic_init_graph_name": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_private" + ].get("dynamic_init_graph_name", None), + "min_word_vocab_freq": self.opt["preprocessing_args"]["min_freq"], + "pretrained_word_emb_name": self.opt["preprocessing_args"]["pretrained_word_emb_name"], + "nlp_processor_args": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["nlp_processor_args"], + "dataitem": Text2TreeDataItem if graph_type != "amr" else AMRDataItem, + #"dataitem": AMRDataItem, + "topology_builder": AMRGraphConstruction if graph_type == "amr" else None, + } + + dataset = EdgeText2TreeDataset(**para_dic) + + self.train_data_loader = DataLoader( + dataset.train, + batch_size=self.opt["training_args"]["batch_size"], + shuffle=True, + num_workers=0, + collate_fn=dataset.collate_fn, + ) + self.test_data_loader = DataLoader( + dataset.test, batch_size=1, shuffle=False, num_workers=0, collate_fn=dataset.collate_fn + ) + self.valid_data_loader = DataLoader( + dataset.val, batch_size=1, shuffle=False, num_workers=0, collate_fn=dataset.collate_fn + ) + self.vocab_model = dataset.vocab_model + self.src_vocab = self.vocab_model.in_word_vocab + self.tgt_vocab = self.vocab_model.out_word_vocab + #self.num_rel = len(dataset.edge_vocab) + #print(dataset.edge_vocab) + self.share_vocab = self.vocab_model.share_vocab if self.use_share_vocab else None + + def _build_model(self): + """For encoder-decoder""" + print(self.opt["model_args"]["graph_embedding_name"]) + if self.opt["model_args"]["graph_embedding_name"] == "rgcn": + if self.opt["model_args"]["graph_construction_name"] == "amr": + self.model = AMRGraph2Tree.from_args(opt=self.opt, vocab_model=self.vocab_model) + else: + self.model = RGCNGraph2Tree.from_args(opt=self.opt, vocab_model=self.vocab_model) + else: + self.model = Graph2Tree.from_args(self.opt, vocab_model=self.vocab_model) + self.model.init(self.opt["training_args"]["init_weight"]) + self.model.to(self.device) + + def _build_optimizer(self): + optim_state = { + "learningRate": self.opt["training_args"]["learning_rate"], + "weight_decay": self.opt["training_args"]["weight_decay"], + } + parameters = [p for p in self.model.parameters() if p.requires_grad] + self.optimizer = optim.Adam( + parameters, lr=optim_state["learningRate"], weight_decay=optim_state["weight_decay"] + ) + + def prepare_ext_vocab(self, batch_graph, src_vocab): + oov_dict = copy.deepcopy(src_vocab) + token_matrix = [] + for n in batch_graph.node_attributes: + node_token = n["token"] + if (n.get("type") is None or n.get("type") == 0) and oov_dict.get_symbol_idx( + node_token + ) == oov_dict.get_symbol_idx(oov_dict.unk_token): + oov_dict.add_symbol(node_token) + token_matrix.append(oov_dict.get_symbol_idx(node_token)) + batch_graph.node_features["token_id_oov"] = torch.tensor(token_matrix, dtype=torch.long).to( + self.device + ) + return oov_dict + + def train_epoch(self, epoch): + loss_to_print = 0 + num_batch = len(self.train_data_loader) + for _, data in tqdm( + enumerate(self.train_data_loader), + desc=f"Epoch {epoch:02d}", + total=len(self.train_data_loader), + ): + batch_graph, batch_tree_list, batch_original_tree_list = ( + data["graph_data"], + data["dec_tree_batch"], + data["original_dec_tree_batch"], + ) + batch_graph = batch_graph.to(self.device) + self.optimizer.zero_grad() + oov_dict = ( + self.prepare_ext_vocab(batch_graph, self.src_vocab) if self.use_copy else None + ) + + if self.use_copy: + batch_tree_list_refined = [] + for item in batch_original_tree_list: + tgt_list = oov_dict.get_symbol_idx_for_list(item.strip().split()) + tgt_tree = Tree.convert_to_tree(tgt_list, 0, len(tgt_list), oov_dict) + batch_tree_list_refined.append(tgt_tree) + loss = self.model( + batch_graph, + batch_tree_list_refined if self.use_copy else batch_tree_list, + oov_dict=oov_dict, + ) + loss.backward() + torch.nn.utils.clip_grad_value_( + self.model.parameters(), self.opt["training_args"]["grad_clip"] + ) + self.optimizer.step() + loss_to_print += loss + return loss_to_print / num_batch + + def train(self): + best_acc = (-1, -1) + best_model = None + + print("-------------\nStarting training.") + for epoch in range(1, self.opt["training_args"]["max_epochs"] + 1): + self.model.train() + loss_to_print = self.train_epoch(epoch) + print("epochs = {}, train_loss = {:.3f}".format(epoch, loss_to_print)) + if epoch > 10 and epoch % 5 == 0: + test_acc = self.eval(self.model, mode="test") + val_acc = self.eval(self.model, mode="val") + if val_acc > best_acc[1]: + best_acc = (test_acc, val_acc) + best_model = self.model + print("Best Acc: {:.3f}\n".format(best_acc[0])) + best_model.save_checkpoint( + self.opt["checkpoint_args"]["out_dir"], self.opt["checkpoint_args"]["checkpoint_name"] + ) + return best_acc + + def eval(self, model, mode="val"): + from examples.pytorch.math_word_problem.mawps.src.evaluation import compute_tree_accuracy + + model.eval() + reference_list = [] + candidate_list = [] + data_loader = self.test_data_loader if mode == "test" else self.valid_data_loader + for data in tqdm(data_loader, desc="Eval: "): + eval_input_graph, _, batch_original_tree_list = ( + data["graph_data"], + data["dec_tree_batch"], + data["original_dec_tree_batch"], + ) + eval_input_graph = eval_input_graph.to(self.device) + oov_dict = self.prepare_ext_vocab(eval_input_graph, self.src_vocab) + + if self.use_copy: + assert len(batch_original_tree_list) == 1 + reference = oov_dict.get_symbol_idx_for_list(batch_original_tree_list[0].split()) + eval_vocab = oov_dict + else: + assert len(batch_original_tree_list) == 1 + reference = model.tgt_vocab.get_symbol_idx_for_list( + batch_original_tree_list[0].split() + ) + eval_vocab = self.tgt_vocab + + candidate = model.translate( + eval_input_graph, + oov_dict=oov_dict, + use_beam_search=True, + beam_size=self.opt["inference_args"]["beam_size"], + ) + candidate = [int(c) for c in candidate] + num_left_paren = sum(1 for c in candidate if eval_vocab.idx2symbol[int(c)] == "(") + num_right_paren = sum(1 for c in candidate if eval_vocab.idx2symbol[int(c)] == ")") + diff = num_left_paren - num_right_paren + if diff > 0: + for _ in range(diff): + candidate.append(self.test_data_loader.tgt_vocab.symbol2idx[")"]) + elif diff < 0: + candidate = candidate[:diff] + # ref_str = convert_to_string(reference, eval_vocab) + # cand_str = convert_to_string(candidate, eval_vocab) + + reference_list.append(reference) + candidate_list.append(candidate) + eval_acc = compute_tree_accuracy(candidate_list, reference_list, eval_vocab) + print("{} accuracy = {:.3f}\n".format(mode, eval_acc)) + return eval_acc + + +################################################################################ +# ArgParse and Helper Functions # +################################################################################ +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-json_config", + "--json_config", + required=True, + type=str, + help="path to the json config file", + ) + args = vars(parser.parse_args()) + + return args + + +def print_config(config): + import pprint + + print("**************** MODEL CONFIGURATION ****************") + pprint.pprint(config) + print("**************** MODEL CONFIGURATION ****************") + + +if __name__ == "__main__": + torch.multiprocessing.set_start_method('spawn') + import platform + import multiprocessing + + if platform.system() == "Darwin": + multiprocessing.set_start_method("spawn") + + cfg = get_args() + config = load_json_config(cfg["json_config"]) + print_config(config) + + start = time.time() + runner = Mawps(config) + best_acc = runner.train() + + end = time.time() + print("total time: {} minutes\n".format((end - start) / 60)) diff --git a/examples/pytorch/math_word_problem/mawps/src_for_amr/utils.py b/examples/pytorch/math_word_problem/mawps/src_for_amr/utils.py new file mode 100644 index 00000000..b932f1d4 --- /dev/null +++ b/examples/pytorch/math_word_problem/mawps/src_for_amr/utils.py @@ -0,0 +1,1125 @@ +from abc import abstractmethod +import copy +import warnings +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from copy import deepcopy +from graph4nlp.pytorch.data.data import GraphData, from_batch +from graph4nlp.pytorch.data.dataset import DataItem, Text2TreeDataset + +from graph4nlp.pytorch.datasets.mawps import MawpsDatasetForTree +from graph4nlp.pytorch.models.graph2tree import Graph2Tree +from graph4nlp.pytorch.modules.graph_embedding_initialization.embedding_construction import BertEmbedding, EmbeddingConstruction, MeanEmbedding, RNNEmbedding, WordEmbedding +from graph4nlp.pytorch.modules.utils.generic_utils import dropout_fn +from graph4nlp.pytorch.modules.utils.tree_utils import Tree + +from examples.pytorch.rgcn.rgcn import RGCN +from graph4nlp.pytorch.modules.utils.vocab_utils import Vocab + +warnings.filterwarnings("ignore") + +class AmrEmbeddingConstruction(EmbeddingConstruction): + """Initial graph embedding construction class. + + Parameters + ---------- + word_vocab : Vocab + The word vocabulary. + single_token_item : bool + Specify whether the item (i.e., node or edge) contains single token or multiple tokens. + emb_strategy : str + Specify the embedding construction strategy including the following options: + - 'w2v': use word2vec embeddings. + - 'w2v_bilstm': use word2vec embeddings, and apply BiLSTM encoders. + - 'w2v_bigru': use word2vec embeddings, and apply BiGRU encoders. + - 'bert': use BERT embeddings. + - 'bert_bilstm': use BERT embeddings, and apply BiLSTM encoders. + - 'bert_bigru': use BERT embeddings, and apply BiGRU encoders. + - 'w2v_bert': use word2vec and BERT embeddings. + - 'w2v_bert_bilstm': use word2vec and BERT embeddings, and apply BiLSTM encoders. + - 'w2v_bert_bigru': use word2vec and BERT embeddings, and apply BiGRU encoders. + Note that if 'w2v' is not applied, `pretrained_word_emb_name` specified in Dataset APIs + will be superseded. + hidden_size : int, optional + The hidden size of RNN layer, default: ``None``. + num_rnn_layers : int, optional + The number of RNN layers, default: ``1``. + fix_word_emb : boolean, optional + Specify whether to fix pretrained word embeddings, default: ``True``. + fix_bert_emb : boolean, optional + Specify whether to fix pretrained BERT embeddings, default: ``True``. + bert_model_name : str, optional + Specify the BERT model name, default: ``'bert-base-uncased'``. + bert_lower_case : bool, optional + Specify whether to lower case the input text for BERT embeddings, default: ``True``. + word_dropout : float, optional + Dropout ratio for word embedding, default: ``None``. + rnn_dropout : float, optional + Dropout ratio for RNN embedding, default: ``None``. + + Note + ---------- + word_emb_type : str or list of str + Specify pretrained word embedding types including "w2v", "node_edge_bert", + or "seq_bert". + node_edge_emb_strategy : str + Specify node/edge embedding strategies including "mean", "bilstm" and "bigru". + seq_info_encode_strategy : str + Specify strategies of encoding sequential information in raw text + data including "none", "bilstm" and "bigru". You might + want to do this in some situations, e.g., when all the nodes are single + tokens extracted from the raw text. + + 1) single-token node (i.e., single_token_item=`True`): + a) 'w2v', 'bert', 'w2v_bert' + b) node_edge_emb_strategy: 'mean' + c) seq_info_encode_strategy: 'none', 'bilstm', 'bigru' + emb_strategy: 'w2v', 'w2v_bilstm', 'w2v_bigru', + 'bert', 'bert_bilstm', 'bert_bigru', + 'w2v_bert', 'w2v_bert_bilstm', 'w2v_bert_bigru' + + 2) multi-token node (i.e., single_token_item=`False`): + a) 'w2v', 'bert', 'w2v_bert' + b) node_edge_emb_strategy: 'mean', 'bilstm', 'bigru' + c) seq_info_encode_strategy: 'none' + emb_strategy: ('w2v', 'w2v_bilstm', 'w2v_bigru', + 'bert', 'bert_bilstm', 'bert_bigru', + 'w2v_bert', 'w2v_bert_bilstm', 'w2v_bert_bigru') + """ + + def __init__( + self, + word_vocab, + single_token_item, + emb_strategy="w2v_bilstm", + hidden_size=None, + num_rnn_layers=1, + fix_word_emb=True, + fix_bert_emb=True, + bert_model_name="bert-base-uncased", + bert_lower_case=True, + word_dropout=None, + bert_dropout=None, + rnn_dropout=None, + ): + super(EmbeddingConstruction, self).__init__() + self.word_dropout = word_dropout + self.bert_dropout = bert_dropout + self.rnn_dropout = rnn_dropout + self.single_token_item = single_token_item + + assert emb_strategy in ( + "w2v", + "w2v_bilstm", + "w2v_bigru", + "bert", + "bert_bilstm", + "bert_bigru", + "w2v_bert", + "w2v_bert_bilstm", + "w2v_bert_bigru", + "w2v_amr", + "w2v_bilstm_amr", + "w2v_bilstm_amr_pos", + ), "emb_strategy must be one of ('w2v', 'w2v_bilstm', 'w2v_bigru', 'bert', 'bert_bilstm', " + "'bert_bigru', 'w2v_bert', 'w2v_bert_bilstm', 'w2v_bert_bigru')" + + word_emb_type = set() + if single_token_item: + node_edge_emb_strategy = None + if "w2v" in emb_strategy: + word_emb_type.add("w2v") + + if "bert" in emb_strategy: + word_emb_type.add("seq_bert") + + if "bilstm" in emb_strategy: + seq_info_encode_strategy = "bilstm" + elif "bigru" in emb_strategy: + seq_info_encode_strategy = "bigru" + else: + seq_info_encode_strategy = "none" + else: + seq_info_encode_strategy = "none" + if "amr" in emb_strategy: + seq_info_encode_strategy = "bilstm" + + if "pos" in emb_strategy: + word_emb_type.add("pos") + #word_emb_type.add("entity_label") + word_emb_type.add("position") + + if "w2v" in emb_strategy: + word_emb_type.add("w2v") + + if "bert" in emb_strategy: + word_emb_type.add("node_edge_bert") + + if "bilstm" in emb_strategy: + node_edge_emb_strategy = "bilstm" + elif "bigru" in emb_strategy: + node_edge_emb_strategy = "bigru" + else: + node_edge_emb_strategy = "mean" + + word_emb_size = 0 + self.word_emb_layers = nn.ModuleDict() + if "w2v" in word_emb_type: + self.word_emb_layers["w2v"] = WordEmbedding( + word_vocab.embeddings.shape[0], + word_vocab.embeddings.shape[1], + pretrained_word_emb=word_vocab.embeddings, + fix_emb=fix_word_emb, + ) + word_emb_size += word_vocab.embeddings.shape[1] + + if "node_edge_bert" in word_emb_type: + self.word_emb_layers["node_edge_bert"] = BertEmbedding( + name=bert_model_name, fix_emb=fix_bert_emb, lower_case=bert_lower_case + ) + word_emb_size += self.word_emb_layers["node_edge_bert"].bert_model.config.hidden_size + + if "seq_bert" in word_emb_type: + self.word_emb_layers["seq_bert"] = BertEmbedding( + name=bert_model_name, fix_emb=fix_bert_emb, lower_case=bert_lower_case + ) + + if node_edge_emb_strategy in ("bilstm", "bigru"): + self.node_edge_emb_layer = RNNEmbedding( + word_emb_size, + hidden_size, + bidirectional=True, + num_layers=num_rnn_layers, + rnn_type="lstm" if node_edge_emb_strategy == "bilstm" else "gru", + dropout=rnn_dropout, + ) + rnn_input_size = hidden_size + elif node_edge_emb_strategy == "mean": + self.node_edge_emb_layer = MeanEmbedding() + rnn_input_size = word_emb_size + else: + rnn_input_size = word_emb_size + + if "pos" in word_emb_type: + self.word_emb_layers["pos"] = WordEmbedding(50, 50) + rnn_input_size += 50 + + if "entity_label" in word_emb_type: + self.word_emb_layers["entity_label"] = WordEmbedding(50, 50) + rnn_input_size += 50 + + if "position" in word_emb_type: + pass + + if "seq_bert" in word_emb_type: + rnn_input_size += self.word_emb_layers["seq_bert"].bert_model.config.hidden_size + + if seq_info_encode_strategy in ("bilstm", "bigru"): + self.output_size = hidden_size + self.seq_info_encode_layer = RNNEmbedding( + rnn_input_size, + hidden_size, + bidirectional=True, + num_layers=num_rnn_layers, + rnn_type="lstm" if seq_info_encode_strategy == "bilstm" else "gru", + dropout=rnn_dropout, + ) + + else: + self.output_size = rnn_input_size + self.seq_info_encode_layer = None + + #self.fc = nn.Linear(376, 300) + + def forward(self, batch_gd): + """Compute initial node/edge embeddings. + + Parameters + ---------- + batch_gd : GraphData + The input graph data. + + Returns + ------- + GraphData + The output graph data with updated node embeddings. + """ + feat = [] + if self.single_token_item: # single-token node graph + token_ids = batch_gd.batch_node_features["token_id"] + if "w2v" in self.word_emb_layers: + word_feat = self.word_emb_layers["w2v"](token_ids).squeeze(-2) + word_feat = dropout_fn( + word_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + feat.append(word_feat) + + else: # multi-token node graph + token_ids = batch_gd.node_features["token_id"] + if "w2v" in self.word_emb_layers: + word_feat = self.word_emb_layers["w2v"](token_ids) + word_feat = dropout_fn( + word_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + feat.append(word_feat) + if any(batch_gd.batch_graph_attributes): + tot = 0 + gd_list = from_batch(batch_gd) + for i, g in enumerate(gd_list): + sentence_id = g.graph_attributes["sentence_id"].to(batch_gd.device) + seq_feat = [] + if "w2v" in self.word_emb_layers: + word_feat = self.word_emb_layers["w2v"](sentence_id) + word_feat = dropout_fn( + word_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + seq_feat.append(word_feat) + else: + RuntimeError("No word embedding layer") + if "pos" in self.word_emb_layers: + sentence_pos = g.graph_attributes["pos_tag_id"].to(batch_gd.device) + pos_feat = self.word_emb_layers["pos"](sentence_pos) + pos_feat = dropout_fn( + pos_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + seq_feat.append(pos_feat) + + if "entity_label" in self.word_emb_layers: + sentence_entity_label = g.graph_attributes["entity_label_id"].to(batch_gd.device) + entity_label_feat = self.word_emb_layers["entity_label"](sentence_entity_label) + entity_label_feat = dropout_fn( + entity_label_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + seq_feat.append(entity_label_feat) + + seq_feat = torch.cat(seq_feat, dim=-1) + + raw_tokens = [dd.strip().split() for dd in g.graph_attributes["sentence"]] + l = [len(s) for s in raw_tokens] + rnn_state = self.seq_info_encode_layer( + seq_feat, torch.LongTensor(l).to(batch_gd.device) + ) + if isinstance(rnn_state, (tuple, list)): + rnn_state = rnn_state[0] + + # update node features + for j in range(g.get_node_num()): + id = g.node_attributes[j]["sentence_id"] + if g.node_attributes[j]["id"] in batch_gd.batch_graph_attributes[i]["mapping"][id]: + rel_list = batch_gd.batch_graph_attributes[i]["mapping"][id][g.node_attributes[j]["id"]] + state = [] + for rel in rel_list: + if rel[1] == "node": + state.append(rnn_state[id][rel[0]]) + # replace embedding of the node + if len(state) > 0: + feat[0][tot + j][0] = torch.stack(state, 0).mean(0) + + tot += g.get_node_num() + + if "node_edge_bert" in self.word_emb_layers: + input_data = [ + batch_gd.node_attributes[i]["token"].strip().split(" ") + for i in range(batch_gd.get_node_num()) + ] + node_edge_bert_feat = self.word_emb_layers["node_edge_bert"](input_data) + node_edge_bert_feat = dropout_fn( + node_edge_bert_feat, self.bert_dropout, shared_axes=[-2], training=self.training + ) + feat.append(node_edge_bert_feat) + + if len(feat) > 0: + feat = torch.cat(feat, dim=-1) + if not any(batch_gd.batch_graph_attributes): + node_token_lens = torch.clamp((token_ids != Vocab.PAD).sum(-1), min=1) + feat = self.node_edge_emb_layer(feat, node_token_lens) + else: + feat = feat.squeeze(dim=1) + if isinstance(feat, (tuple, list)): + feat = feat[-1] + + feat = batch_gd.split_features(feat) + + if (self.seq_info_encode_layer is None and "seq_bert" not in self.word_emb_layers) or any(batch_gd.batch_graph_attributes): + if isinstance(feat, list): + feat = torch.cat(feat, -1) + + batch_gd.batch_node_features["node_feat"] = feat + + return batch_gd + else: # single-token node graph + new_feat = feat + if "seq_bert" in self.word_emb_layers: + gd_list = from_batch(batch_gd) + raw_tokens = [ + [gd.node_attributes[i]["token"] for i in range(gd.get_node_num())] + for gd in gd_list + ] + bert_feat = self.word_emb_layers["seq_bert"](raw_tokens) + bert_feat = dropout_fn( + bert_feat, self.bert_dropout, shared_axes=[-2], training=self.training + ) + new_feat.append(bert_feat) + + new_feat = torch.cat(new_feat, -1) + if self.seq_info_encode_layer is None: + batch_gd.batch_node_features["node_feat"] = new_feat + + return batch_gd + + rnn_state = self.seq_info_encode_layer( + new_feat, torch.LongTensor(batch_gd._batch_num_nodes).to(batch_gd.device) + ) + if isinstance(rnn_state, (tuple, list)): + rnn_state = rnn_state[0] + + batch_gd.batch_node_features["node_feat"] = rnn_state + + return batch_gd + +class AMRGraphEmbeddingInitialization(nn.Module): + def __init__( + self, + word_vocab, + embedding_style, + hidden_size=None, + fix_word_emb=True, + fix_bert_emb=True, + word_dropout=None, + rnn_dropout=None, + ): + super(AMRGraphEmbeddingInitialization, self).__init__() + self.embedding_layer = AmrEmbeddingConstruction( + word_vocab, + embedding_style["single_token_item"], + emb_strategy=embedding_style["emb_strategy"], + hidden_size=hidden_size, + num_rnn_layers=embedding_style.get("num_rnn_layers", 1), + fix_word_emb=fix_word_emb, + fix_bert_emb=fix_bert_emb, + bert_model_name=embedding_style.get("bert_model_name", "bert-base-uncased"), + bert_lower_case=embedding_style.get("bert_lower_case", True), + word_dropout=word_dropout, + rnn_dropout=rnn_dropout, + ) + + @abstractmethod + def forward(self, graph_data: GraphData): + return self.embedding_layer(graph_data) +class AMRDataItem(DataItem): + def __init__(self, input_text, output_text, tokenizer, output_tree=None, share_vocab=True): + super(AMRDataItem, self).__init__(input_text, tokenizer) + self.output_text = output_text + self.share_vocab = share_vocab + self.output_tree = output_tree + + def extract(self): + """ + Returns + ------- + Input tokens and output tokens + """ + g: GraphData = self.graph + + input_tokens = [] + for i in range(g.get_node_num()): + tokenized_token = self.tokenizer(g.node_attributes[i]["token"]) + input_tokens.extend(tokenized_token) + + for s in g.graph_attributes["sentence"]: + input_tokens.extend(s.strip().split(" ")) + + output_tokens = self.tokenizer(self.output_text) + + return input_tokens, output_tokens + + def extract_edge_tokens(self): + g: GraphData = self.graph + edge_tokens = [] + for i in range(g.get_edge_num()): + edge_tokens.append(g.edge_attributes[i]["token"]) + return edge_tokens + +class RGCNGraph2Tree(Graph2Tree): + def __init__( + self, + vocab_model, + embedding_style, + graph_construction_name, + # embedding + emb_input_size, + emb_hidden_size, + emb_word_dropout, + emb_rnn_dropout, + emb_fix_word_emb, + emb_fix_bert_emb, + # gnn + gnn, + gnn_num_layers, + gnn_direction_option, + gnn_input_size, + gnn_hidden_size, + gnn_output_size, + gnn_feat_drop, + gnn_attn_drop, + # decoder + dec_use_copy, + dec_hidden_size, + dec_dropout, + dec_teacher_forcing_rate, + dec_max_decoder_step, + dec_max_tree_depth, + dec_attention_type, + dec_use_sibling, + # optional + criterion=None, + share_vocab=False, + **kwargs + ): + super(RGCNGraph2Tree, self).__init__( + vocab_model=vocab_model, + embedding_style=embedding_style, + graph_construction_name=graph_construction_name, + # embedding + emb_input_size=emb_input_size, + emb_hidden_size=emb_hidden_size, + emb_word_dropout=emb_word_dropout, + emb_rnn_dropout=emb_rnn_dropout, + emb_fix_word_emb=emb_fix_word_emb, + emb_fix_bert_emb=emb_fix_bert_emb, + # gnn + gnn=gnn, + gnn_num_layers=gnn_num_layers, + gnn_direction_option=gnn_direction_option, + gnn_input_size=gnn_input_size, + gnn_hidden_size=gnn_hidden_size, + gnn_output_size=gnn_output_size, + gnn_feat_drop=gnn_feat_drop, + gnn_attn_drop=gnn_attn_drop, + # decoder + dec_use_copy=dec_use_copy, + dec_hidden_size=dec_hidden_size, + dec_dropout=dec_dropout, + dec_teacher_forcing_rate=dec_teacher_forcing_rate, + dec_max_decoder_step=dec_max_decoder_step, + dec_max_tree_depth=dec_max_tree_depth, + dec_attention_type=dec_attention_type, + dec_use_sibling=dec_use_sibling, + # optional + criterion=criterion, + share_vocab=share_vocab, + **kwargs + ) + + def _build_gnn_encoder( + self, + gnn, + num_layers, + input_size, + hidden_size, + output_size, + direction_option, + feats_dropout, + gnn_heads=None, + gnn_residual=True, + gnn_attn_dropout=0.0, + gnn_activation=F.relu, # gat + gnn_bias=True, + gnn_allow_zero_in_degree=True, + gnn_norm="both", + gnn_weight=True, + gnn_use_edge_weight=False, + gnn_gcn_norm="both", # gcn + gnn_n_etypes=1, # ggnn + gnn_aggregator_type="lstm", # graphsage + **kwargs + ): + if gnn == "rgcn": + self.gnn_encoder = RGCN( + num_layers, + input_size, + hidden_size, + output_size, + num_rels=80, + num_bases=4, + gpu=0, + ) + else: + raise NotImplementedError() + + @classmethod + def from_args(cls, opt, vocab_model): + """ + The function for building ``Graph2Tree`` model. + Parameters + ---------- + opt: dict + The configuration dict. It should has the same hierarchy and keys as the template. + vocab_model: VocabModel + The vocabulary. + + Returns + ------- + model: Graph2Tree + """ + initializer_args = cls._get_node_initializer_params(opt) + gnn_args = cls._get_gnn_params(opt) + dec_args = cls._get_decoder_params(opt) + + args = copy.deepcopy(initializer_args) + args.update(gnn_args) + args.update(dec_args) + args["share_vocab"] = opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ][ + "share_vocab" + ] # noqa + return cls(vocab_model=vocab_model, **args) + +class AMRGraph2Tree(RGCNGraph2Tree): + def __init__( + self, + vocab_model, + embedding_style, + graph_construction_name, + # embedding + emb_input_size, + emb_hidden_size, + emb_word_dropout, + emb_rnn_dropout, + emb_fix_word_emb, + emb_fix_bert_emb, + # gnn + gnn, + gnn_num_layers, + gnn_direction_option, + gnn_input_size, + gnn_hidden_size, + gnn_output_size, + gnn_feat_drop, + gnn_attn_drop, + # decoder + dec_use_copy, + dec_hidden_size, + dec_dropout, + dec_teacher_forcing_rate, + dec_max_decoder_step, + dec_max_tree_depth, + dec_attention_type, + dec_use_sibling, + # optional + criterion=None, + share_vocab=False, + **kwargs + ): + style = embedding_style["emb_strategy"] + embedding_style["emb_strategy"] = "w2v_bilstm" + super(RGCNGraph2Tree, self).__init__( + vocab_model=vocab_model, + embedding_style=embedding_style, + graph_construction_name=graph_construction_name, + # embedding + emb_input_size=emb_input_size, + emb_hidden_size=emb_hidden_size, + emb_word_dropout=emb_word_dropout, + emb_rnn_dropout=emb_rnn_dropout, + emb_fix_word_emb=emb_fix_word_emb, + emb_fix_bert_emb=emb_fix_bert_emb, + # gnn + gnn=gnn, + gnn_num_layers=gnn_num_layers, + gnn_direction_option=gnn_direction_option, + gnn_input_size=gnn_input_size, + gnn_hidden_size=gnn_hidden_size, + gnn_output_size=gnn_output_size, + gnn_feat_drop=gnn_feat_drop, + gnn_attn_drop=gnn_attn_drop, + # decoder + dec_use_copy=dec_use_copy, + dec_hidden_size=dec_hidden_size, + dec_dropout=dec_dropout, + dec_teacher_forcing_rate=dec_teacher_forcing_rate, + dec_max_decoder_step=dec_max_decoder_step, + dec_max_tree_depth=dec_max_tree_depth, + dec_attention_type=dec_attention_type, + dec_use_sibling=dec_use_sibling, + # optional + criterion=criterion, + share_vocab=share_vocab, + **kwargs + ) + embedding_style["emb_strategy"] = style + self.graph_initializer = AMRGraphEmbeddingInitialization( + word_vocab=vocab_model.in_word_vocab, + embedding_style=embedding_style, + hidden_size=emb_hidden_size, + word_dropout=emb_word_dropout, + rnn_dropout=emb_rnn_dropout, + fix_word_emb=emb_fix_word_emb, + fix_bert_emb=emb_fix_bert_emb, + ) +class InferenceText2TreeDataset(Text2TreeDataset): + def __init__( + self, + graph_construction_name: str, + root_dir: str = None, + static_or_dynamic: str = "static", + topology_builder = None, + topology_subdir: str = None, + dynamic_init_graph_name: str = None, + dynamic_init_topology_builder = None, + dynamic_init_topology_aux_args=None, + share_vocab=True, + dataitem=None, + init_edge_vocab=True, + is_hetero=True, + **kwargs, + ): + super(InferenceText2TreeDataset, self).__init__( + root_dir=root_dir, + graph_construction_name=graph_construction_name, + topology_builder=topology_builder, + topology_subdir=topology_subdir, + static_or_dynamic=static_or_dynamic, + share_vocab=share_vocab, + dynamic_init_topology_builder=dynamic_init_topology_builder, + dynamic_init_topology_aux_args=dynamic_init_topology_aux_args, + init_edge_vocab=init_edge_vocab, + is_hetero=True, + **kwargs, + ) + self.data_item_type = dataitem + + def parse_file(self, file_path) -> list: + """ + Read and parse the file specified by `file_path`. The file format is specified by + each individual task-specific base class. Returns all the indices of data items + in this file w.r.t. the whole dataset. + + For Text2TreeDataset, the format of the input file should contain lines of input, + each line representing one record of data. The input and output is separated by + a tab(\t). + + Examples + -------- + input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 ) + + DataItem: input_text="list job use languageid0", output_text="job ( ANS ) , + language ( ANS , languageid0 )" + + Parameters + ---------- + file_path: str + The path of the input file. + + Returns + ------- + list + The indices of data items in the file w.r.t. the whole dataset. + """ + data = [] + with open(file_path, "r") as f: + lines = f.readlines() + for line in lines: + input, output = line.split("\t") + data_item = self.data_item_type( + input_text=input, + output_text=output, + output_tree=None, + tokenizer=self.tokenizer, + share_vocab=self.share_vocab, + ) + data.append(data_item) + return data + + def vectorization(self, data_items): + """For tree decoder we also need the vectorize the tree output.""" + for item in data_items: + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = self.vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + + token_matrix = [] + if self.init_edge_vocab: + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = self.vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + if "pos_tag" in graph.graph_attributes: + pos_vocab = [".", "CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNP", "NNPS", "NNS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"] + pos_map = {pos: i for i, pos in enumerate(pos_vocab)} + maxlen = max(len(pos_tag) for pos_tag in graph.graph_attributes["pos_tag"]) + pos_token_id = torch.zeros(len(graph.graph_attributes["pos_tag"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["pos_tag"]): + for j, token in enumerate(sentence_token): + if token in pos_map: + pos_token_id[i][j] = pos_map[token] + else: + print('pos_tag', token) + graph.graph_attributes["pos_tag_id"] = pos_token_id + + if "entity_label" in graph.graph_attributes: + entity_label = ["O", "PERSON", "LOCATION", "ORGANIZATION", "ORGANIZATION", "MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET", "EMAIL", "URL", "CITY", "STATE_OR_PROVINCE", "COUNTRY", "NATIONALITY", "RELIGION", "TITLE", "IDEOLOGY", "CRIMINAL_CHARGE", "CAUSE_OF_DEATH", "HANDLE"] + entity_map = {entity: i for i, entity in enumerate(entity_label)} + maxlen = max(len(entity_tag) for entity_tag in graph.graph_attributes["entity_label"]) + entity_token_id = torch.zeros(len(graph.graph_attributes["entity_label"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["entity_label"]): + for j, token in enumerate(sentence_token): + if token in entity_map: + entity_token_id[i][j] = entity_map[token] + else: + print('entity_label', token) + graph.graph_attributes["entity_label_id"] = entity_token_id + + if "sentence" in graph.graph_attributes: + maxlen = max(len(sentence.strip().split()) for sentence in graph.graph_attributes["sentence"]) + seq_token_id = torch.zeros(len(graph.graph_attributes["sentence"]), maxlen, dtype=torch.long) + for i, sentence in enumerate(graph.graph_attributes["sentence"]): + sentence_token = sentence.strip().split() + for j, token in enumerate(sentence_token): + seq_token_id[i][j] = self.vocab_model.in_word_vocab.get_symbol_idx(token) + graph.graph_attributes["sentence_id"] = seq_token_id + + tgt = item.output_text + tgt_list = self.vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), self.vocab_model.out_word_vocab + ) + item.output_tree = output_tree + + def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): + item = deepcopy(data_item) + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + if hasattr(vocab_model, "edge_vocab"): + token_matrix = [] + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + + if "pos_tag" in graph.graph_attributes: + pos_vocab = [".", "CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNP", "NNPS", "NNS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"] + pos_map = {pos: i for i, pos in enumerate(pos_vocab)} + maxlen = max(len(pos_tag) for pos_tag in graph.graph_attributes["pos_tag"]) + pos_token_id = torch.zeros(len(graph.graph_attributes["pos_tag"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["pos_tag"]): + for j, token in enumerate(sentence_token): + if token in pos_map: + pos_token_id[i][j] = pos_map[token] + else: + print('pos_tag', token) + graph.graph_attributes["pos_tag_id"] = pos_token_id + + if "entity_label" in graph.graph_attributes: + entity_label = ["O", "PERSON", "LOCATION", "ORGANIZATION", "ORGANIZATION", "MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET", "EMAIL", "URL", "CITY", "STATE_OR_PROVINCE", "COUNTRY", "NATIONALITY", "RELIGION", "TITLE", "IDEOLOGY", "CRIMINAL_CHARGE", "CAUSE_OF_DEATH", "HANDLE"] + entity_map = {entity: i for i, entity in enumerate(entity_label)} + maxlen = max(len(entity_tag) for entity_tag in graph.graph_attributes["entity_label"]) + entity_token_id = torch.zeros(len(graph.graph_attributes["entity_label"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["entity_label"]): + for j, token in enumerate(sentence_token): + if token in entity_map: + entity_token_id[i][j] = entity_map[token] + else: + print('entity_label', token) + graph.graph_attributes["entity_label_id"] = entity_token_id + + if "sentence" in graph.graph_attributes: + maxlen = max(len(sentence.strip().split()) for sentence in graph.graph_attributes["sentence"]) + seq_token_id = torch.zeros(len(graph.graph_attributes["sentence"]), maxlen, dtype=torch.long) + for i, sentence in enumerate(graph.graph_attributes["sentence"]): + sentence_token = sentence.strip().split() + for j, token in enumerate(sentence_token): + seq_token_id[i][j] = vocab_model.in_word_vocab.get_symbol_idx(token) + graph.graph_attributes["sentence_id"] = seq_token_id + + if isinstance(item.output_text, str): + tgt = item.output_text + tgt_list = vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), vocab_model.out_word_vocab + ) + item.output_tree = output_tree + return item + + +class EdgeText2TreeDataset(MawpsDatasetForTree): + def __init__( + self, + root_dir, + # topology_builder, + topology_subdir, + graph_construction_name, + static_or_dynamic="static", + topology_builder=None, + merge_strategy="tailhead", + edge_strategy=None, + dynamic_init_graph_name=None, + dynamic_init_topology_builder=None, + dynamic_init_topology_aux_args=None, + nlp_processor_args=None, + # pretrained_word_emb_file=None, + pretrained_word_emb_name="6B", + pretrained_word_emb_url=None, + pretrained_word_emb_cache_dir=None, + val_split_ratio=0, + word_emb_size=300, + share_vocab=True, + enc_emb_size=300, + dec_emb_size=300, + min_word_vocab_freq=1, + max_word_vocab_size=100000, + for_inference=False, + reused_vocab_model=None, + dataitem=None, + init_edge_vocab=True, + is_hetero=True, + ): + """ + Parameters + ---------- + root_dir: str + The path of dataset. + graph_name: str + The name of graph construction method. E.g., "dependency". + Note that if it is in the provided graph names (i.e., "dependency", \ + "constituency", "ie", "node_emb", "node_emb_refine"), the following \ + parameters are set by default and users can't modify them: + 1. ``topology_builder`` + 2. ``static_or_dynamic`` + If you need to customize your graph construction method, you should rename the \ + ``graph_name`` and set the parameters above. + topology_builder: GraphConstructionBase + The graph construction class. + topology_subdir: str + The directory name of processed path. + static_or_dynamic: str, default='static' + The graph type. Expected in ('static', 'dynamic') + edge_strategy: str, default=None + The edge strategy. Expected in (None, 'homogeneous', 'as_node'). + If set `None`, it will be 'homogeneous'. + merge_strategy: str, default=None + The strategy to merge sub-graphs. Expected in (None, 'tailhead', 'user_define'). + If set `None`, it will be 'tailhead'. + share_vocab: bool, default=False + Whether to share the input vocabulary with the output vocabulary. + dynamic_init_graph_name: str, default=None + The graph name of the initial graph. Expected in (None, "line", "dependency", \ + "constituency"). + Note that if it is in the provided graph names (i.e., "line", "dependency", \ + "constituency"), the following parameters are set by default and users \ + can't modify them: + 1. ``dynamic_init_topology_builder`` + If you need to customize your graph construction method, you should rename the \ + ``graph_name`` and set the parameters above. + dynamic_init_topology_builder: GraphConstructionBase + The graph construction class. + dynamic_init_topology_aux_args: None, + TBD. + """ + # Initialize the dataset. If the preprocessed files are not found, + # then do the preprocessing and save them. + super(EdgeText2TreeDataset, self).__init__( + root_dir=root_dir, + topology_builder=topology_builder, + topology_subdir=topology_subdir, + graph_construction_name=graph_construction_name, + static_or_dynamic=static_or_dynamic, + edge_strategy=edge_strategy, + merge_strategy=merge_strategy, + share_vocab=share_vocab, + pretrained_word_emb_name=pretrained_word_emb_name, + val_split_ratio=val_split_ratio, + word_emb_size=word_emb_size, + dynamic_init_graph_name=dynamic_init_graph_name, + dynamic_init_topology_builder=dynamic_init_topology_builder, + dynamic_init_topology_aux_args=dynamic_init_topology_aux_args, + nlp_processor_args=nlp_processor_args, + enc_emb_size=enc_emb_size, + dec_emb_size=dec_emb_size, + min_word_vocab_freq=min_word_vocab_freq, + max_word_vocab_size=max_word_vocab_size, + for_inference=for_inference, + reused_vocab_model=reused_vocab_model, + init_edge_vocab=init_edge_vocab, + is_hetero=is_hetero, + ) + self.data_item_type = dataitem + + @property + def processed_file_names(self): + """At least 2 reserved keys should be fiiled: 'vocab', 'data'.""" + return {"vocab": "vocab.pt", "data": "data.pt"} + + def parse_file(self, file_path) -> list: + """ + Read and parse the file specified by `file_path`. The file format is specified by + each individual task-specific base class. Returns all the indices of data items + in this file w.r.t. the whole dataset. + + For Text2TreeDataset, the format of the input file should contain lines of input, + each line representing one record of data. The input and output is separated by + a tab(\t). + + Examples + -------- + input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 ) + + DataItem: input_text="list job use languageid0", output_text="job ( ANS ) , + language ( ANS , languageid0 )" + + Parameters + ---------- + file_path: str + The path of the input file. + + Returns + ------- + list + The indices of data items in the file w.r.t. the whole dataset. + """ + data = [] + with open(file_path, "r") as f: + lines = f.readlines() + for line in lines: + input, output = line.split("\t") + data_item = self.data_item_type( + input_text=input, + output_text=output, + output_tree=None, + tokenizer=self.tokenizer, + share_vocab=self.share_vocab, + ) + data.append(data_item) + return data + + def vectorization(self, data_items): + """For tree decoder we also need the vectorize the tree output.""" + for item in data_items: + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = self.vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + + token_matrix = [] + if self.is_hetero: + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = self.vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + if "pos_tag" in graph.graph_attributes: + pos_vocab = [".", "CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNP", "NNPS", "NNS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"] + pos_map = {pos: i for i, pos in enumerate(pos_vocab)} + maxlen = max(len(pos_tag) for pos_tag in graph.graph_attributes["pos_tag"]) + pos_token_id = torch.zeros(len(graph.graph_attributes["pos_tag"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["pos_tag"]): + for j, token in enumerate(sentence_token): + if token in pos_map: + pos_token_id[i][j] = pos_map[token] + else: + print('pos_tag', token) + graph.graph_attributes["pos_tag_id"] = pos_token_id + + if "entity_label" in graph.graph_attributes: + entity_label = ["O", "PERSON", "LOCATION", "ORGANIZATION", "ORGANIZATION", "MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET", "EMAIL", "URL", "CITY", "STATE_OR_PROVINCE", "COUNTRY", "NATIONALITY", "RELIGION", "TITLE", "IDEOLOGY", "CRIMINAL_CHARGE", "CAUSE_OF_DEATH", "HANDLE"] + entity_map = {entity: i for i, entity in enumerate(entity_label)} + maxlen = max(len(entity_tag) for entity_tag in graph.graph_attributes["entity_label"]) + entity_token_id = torch.zeros(len(graph.graph_attributes["entity_label"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["entity_label"]): + for j, token in enumerate(sentence_token): + if token in entity_map: + entity_token_id[i][j] = entity_map[token] + else: + print('entity_label', token) + graph.graph_attributes["entity_label_id"] = entity_token_id + + if "sentence" in graph.graph_attributes: + maxlen = max(len(sentence.strip().split()) for sentence in graph.graph_attributes["sentence"]) + seq_token_id = torch.zeros(len(graph.graph_attributes["sentence"]), maxlen, dtype=torch.long) + for i, sentence in enumerate(graph.graph_attributes["sentence"]): + sentence_token = sentence.strip().split() + for j, token in enumerate(sentence_token): + seq_token_id[i][j] = self.vocab_model.in_word_vocab.get_symbol_idx(token) + graph.graph_attributes["sentence_id"] = seq_token_id + + tgt = item.output_text + tgt_list = self.vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), self.vocab_model.out_word_vocab + ) + item.output_tree = output_tree + + def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): + item = deepcopy(data_item) + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + # test if vocab_model has edge_vocab attribute + if hasattr(vocab_model, "edge_vocab"): + token_matrix = [] + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + if isinstance(item.output_text, str): + tgt = item.output_text + tgt_list = vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), vocab_model.out_word_vocab + ) + item.output_tree = output_tree + return item \ No newline at end of file diff --git a/examples/pytorch/rgcn/__init__.py b/examples/pytorch/rgcn/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/pytorch/rgcn/main.py b/examples/pytorch/rgcn/main.py new file mode 100644 index 00000000..686231b2 --- /dev/null +++ b/examples/pytorch/rgcn/main.py @@ -0,0 +1,188 @@ +import argparse +import torch +import dgl +import time +import torch.nn.functional as F +from torchmetrics.functional import accuracy +from rgcn import RGCN +from dgl.data.rdf import AIFBDataset, MUTAGDataset, BGSDataset, AMDataset +from graph4nlp.pytorch.data.data import from_dgl + +# Fix random seed +# torch.manual_seed(1024) +# import random +# random.seed(1024) +# import numpy as np +# np.random.seed(1024) + +# Load dataset +# Reference: dgl/examples/pytorch/rgcn/entity_utils.py (https://github.com/dmlc/dgl/blob/master/examples/pytorch/rgcn/entity_utils.py) +def load_data(data_name='aifb', get_norm=False, inv_target=False): + if data_name == 'aifb': + dataset = AIFBDataset() + # Test Accuracy: + # 0.9444, 0.8889, 0.9722, 0.9167, 0.9444 without enorm. + # 0.8611, 0.8889, 0.8889, 0.8889, 0.8333 + # avg: 0.93332 (without enorm) + # avg: 0.87222 + # DGL: 0.8889, 0.8889, 0.8056, 0.8889, 0.8611 + # DGL avg: 0.86668 + # paper: 0.9583 + # note: Could stuck at Local minimum of train loss between 0.2-0.35. + elif data_name == 'mutag': + dataset = MUTAGDataset() + # Test Accuracy: + # 0.6912, 0.7500, 0.7353, 0.6324, 0.7353 + # avg: 0.68884 + # DGL: 0.6765, 0.7059, 0.7353, 0.6765, 0.6912 + # DGL avg: 0.69724 + # paper: 0.7323 + # note: Could stuck at local minimum of train acc: 0.3897 & loss 0.6931 + elif data_name == 'bgs': + dataset = BGSDataset() + # Test Accuracy: + # 0.8966, 0.9310, 0.8966, 0.7931, 0.8621 + # avg: 0.87588 + # DGL: 0.7931, 0.9310, 0.8966, 0.8276, 0.8966 + # DGL avg: 0.86898 + # paper: 0.8310 + # note: Could stuck at local minimum of train acc: 0.6325 & loss: 0.6931 + else: + dataset = AMDataset() + # Test Accuracy: + # 0.7525, 0.7374, 0.7424, 0.7424, 0.7424 + # avg: 0.74342 + # DGL: 0.7677, 0.7677, 0.7323, 0.7879, 0.7677 + # DGL avg: 0.76466 + # paper: 0.8929 + # note: args.hidden_size is 10. + # Could stuck at local minimum of train loss: 0.3-0.5 + + # Load hetero-graph + hg = dataset[0] + + num_rels = len(hg.canonical_etypes) + category = dataset.predict_category + num_classes = dataset.num_classes + labels = hg.nodes[category].data.pop('labels') + train_mask = hg.nodes[category].data.pop('train_mask') + test_mask = hg.nodes[category].data.pop('test_mask') + train_idx = torch.nonzero(train_mask, as_tuple=False).squeeze() + test_idx = torch.nonzero(test_mask, as_tuple=False).squeeze() + + if get_norm: + # Calculate normalization weight for each edge, + # 1. / d, d is the degree of the destination node + for cetype in hg.canonical_etypes: + hg.edges[cetype].data['norm'] = dgl.norm_by_dst(hg, cetype).unsqueeze(1) + edata = ['norm'] + else: + edata = None + category_id = hg.ntypes.index(category) + g = dgl.to_homogeneous(hg, edata=edata) + node_ids = torch.arange(g.num_nodes()) + + # find out the target node ids in g + loc = (g.ndata['_TYPE'] == category_id) + target_idx = node_ids[loc] + + if inv_target: + # Map global node IDs to type-specific node IDs. This is required for + # looking up type-specific labels in a minibatch + inv_target = torch.empty((g.num_nodes(),), dtype=torch.int64) + inv_target[target_idx] = torch.arange(0, target_idx.shape[0], + dtype=inv_target.dtype) + return g, num_rels, num_classes, labels, train_idx, test_idx, target_idx, inv_target + else: + return g, num_rels, num_classes, labels, train_idx, test_idx, target_idx + + +def main(args): + g, num_rels, num_classes, labels, train_idx, test_idx, target_idx = load_data(data_name=args.dataset, get_norm=True) + + graph = from_dgl(g, is_hetero=False) + num_nodes = graph.get_node_num() + emb = torch.nn.Embedding(num_nodes, args.hidden_size) + # emb.requires_grad = True + graph.node_features['node_feat'] = emb.weight + + model = RGCN(num_layers=args.num_hidden_layers, + input_size=args.hidden_size, + hidden_size=args.hidden_size, + output_size=num_classes, + num_rels=num_rels, + num_bases=args.num_bases, + use_self_loop=args.use_self_loop, + gpu=args.gpu, + dropout = args.dropout) + optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) + print("start training...") + model.train() + for epoch in range(args.num_epochs): + logits = model(graph).node_features["node_emb"] + logits = logits[target_idx] + loss = F.cross_entropy(logits[train_idx], labels[train_idx]) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + train_acc = accuracy(logits[train_idx].argmax(dim=1), labels[train_idx]).item() + print("Epoch {:05d} | Train Accuracy: {:.4f} | Train Loss: {:.4f}".format(epoch, train_acc, loss.item())) + print() + # Save Model + # torch.save(model.state_dict(), "./rgcn_model.pt") + print("start evaluating...") + model.eval() + with torch.no_grad(): + logits = model(graph).node_features["node_emb"] + logits = logits[target_idx] + test_acc = accuracy(logits[test_idx].argmax(dim=1), labels[test_idx]).item() + print("Test Accuracy: {:.4f}".format(test_acc)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='RGCN for entity classification') + parser.add_argument("--num-hidden-layers", type=int, default=1, + help="number of hidden layers beside input/output layer") + parser.add_argument("--hidden-size", type=int, default=16, + help="dimension of hidden layer") + parser.add_argument("--gpu", type=int, default=-1, + help="GPU device number, -1 for cpu") + parser.add_argument("--num-bases", type=int, default=-1, + help="number of filter weight matrices, default: -1 [use all]") + parser.add_argument("-d", "--dataset", type=str, required=True, + choices=['aifb', 'mutag', 'bgs', 'am'], + help="dataset to use") + parser.add_argument("--use-self-loop", type=bool, default=False, + help="Consider self-loop edges or not") + parser.add_argument("--dropout", type=float, default=0.0, + help="Dropout rate") + parser.add_argument("--lr", type=float, default=1e-2, + help="Start learning rate") + parser.add_argument("--wd", type=float, default=5e-4, + help="weight decay") + parser.add_argument("--num-epochs", type=int, default=50, + help="Number of training epochs") + + args = parser.parse_args() + print(args) + main(args) + + + + + +"""Deprecated RGCN code on Heterogeneous graph due to +the lack of support from data structure. The following supports +are needed (but not limit to): +- Redefine the feature data structure of node/edge + - Index node/edge ids by their type. + - Enable type indexed features. +- Make corresponding changes on views. +- Make corresponding changes on set/get features functions. + +This example bypasses it by storing the features in the model +itself. It is a code trick and therefore not recommended to +the user. +""" \ No newline at end of file diff --git a/examples/pytorch/rgcn/rgcn.py b/examples/pytorch/rgcn/rgcn.py index 25291e33..e24ec46d 100644 --- a/examples/pytorch/rgcn/rgcn.py +++ b/examples/pytorch/rgcn/rgcn.py @@ -1,9 +1,13 @@ -import dgl +import warnings +import dgl.function as fn +import torch import torch.nn as nn import torch.nn.functional as F +import dgl +from dgl.utils import check_eq_shape, expand_as_pair from dgl.nn.pytorch import RelGraphConv -from .base import GNNBase, GNNLayerBase +from graph4nlp.pytorch.modules.graph_embedding_learning.base import GNNBase, GNNLayerBase class RGCN(GNNBase): @@ -18,19 +22,18 @@ class RGCN(GNNBase): Number of RGCN layers. input_size : int, or pair of ints Input feature size. - hidden_size: int list of int + hidden_size: int Hidden layer size. - If a scalar is given, the sizes of all the hidden layers are the same. - If a list of scalar is given, each element in the list is the size of each hidden layer. - Example: [100,50] output_size : int Output feature size. num_rels : int Number of relations. num_bases : int, optional - Number of bases. Needed when ``regularizer`` is specified. Default: ``None``. + Number of bases. Needed when ``regularizer`` is specified. Default: ``-1`` [all]. use_self_loop : bool, optional - True to include self loop message. Default: ``True``. + True to include self loop message. Default: ``False``. + gpu : int, optional + True to use gpu. Default: ``-1`` [cpu]. dropout : float, optional Dropout rate. Default: ``0.0`` """ @@ -42,58 +45,72 @@ def __init__( hidden_size, output_size, num_rels, - num_bases=None, + num_bases=-1, use_self_loop=True, + gpu=False, dropout=0.0, + ): super(RGCN, self).__init__() self.num_layers = num_layers + if num_bases == -1: + num_bases = num_rels self.num_rels = num_rels self.num_bases = num_bases self.use_self_loop = use_self_loop - self.dropout = dropout + self.dropout = nn.Dropout(dropout) + self.gpu = gpu self.RGCN_layers = nn.ModuleList() - - # transform the hidden size format - if self.num_layers > 1 and type(hidden_size) is int: - hidden_size = [hidden_size for i in range(self.num_layers - 1)] - + # input layers: + self.RGCN_layers.append( + RGCNLayer( + input_size, + hidden_size, + num_rels=self.num_rels, + regularizer="basis", + num_bases=self.num_bases, + bias=True, + activation=F.relu, + self_loop=self.use_self_loop, + dropout=dropout + ) + ) if self.num_layers > 1: # input projection self.RGCN_layers.append( RGCNLayer( - input_size, - hidden_size[0], + hidden_size, + hidden_size, num_rels=self.num_rels, regularizer="basis", num_bases=self.num_bases, bias=True, activation=F.relu, self_loop=self.use_self_loop, - dropout=self.dropout, + dropout=dropout ) ) + # hidden layers - for l in range(1, self.num_layers - 1): - # due to multi-head, the input_size = hidden_size * num_heads + for l in range(1, self.num_layers-1): self.RGCN_layers.append( RGCNLayer( - hidden_size[l - 1], - hidden_size[l], + hidden_size, + hidden_size, num_rels=self.num_rels, regularizer="basis", num_bases=self.num_bases, bias=True, activation=F.relu, self_loop=self.use_self_loop, - dropout=self.dropout, + dropout=dropout ) ) # output projection self.RGCN_layers.append( RGCNLayer( - hidden_size[-1] if self.num_layers > 1 else input_size, + hidden_size, output_size, num_rels=self.num_rels, regularizer="basis", @@ -101,10 +118,13 @@ def __init__( bias=True, activation=F.relu, self_loop=self.use_self_loop, - dropout=self.dropout, + dropout=dropout ) ) + if self.gpu != -1: + self.to(device=self.gpu) + def forward(self, graph): r"""Compute RGCN layer. @@ -122,18 +142,19 @@ def forward(self, graph): named as "node_emb". """ - h = graph.node_features["node_feat"] - # get the node feature tensor from graph - g = graph.to_dgl() # transfer the current NLPgraph to DGL graph - edge_type = g.edata[dgl.ETYPE].long() - # output projection - if self.num_layers > 1: - for l in range(0, self.num_layers - 1): - h = self.RGCN_layers[l](g, h, edge_type) - + # transfer the current NLPgraph to DGL graph + g = graph.to_dgl() + h = graph.node_features['node_feat'] + edge_type = graph.edge_features['token_id'].squeeze(1) + for l in range(self.num_layers): + h = self.RGCN_layers[l](g, h, edge_type) + h = self.dropout(F.relu(h)) logits = self.RGCN_layers[-1](g, h, edge_type) + + # put the results into the NLPGraph + # graph.node_features['node_feat'] = h + graph.node_features["node_emb"] = logits - graph.node_features["node_emb"] = logits # put the results into the NLPGraph return graph @@ -176,26 +197,25 @@ def __init__( output_size, num_rels, regularizer=None, - num_bases=None, + num_bases=-1, bias=True, activation=None, self_loop=False, dropout=0.0, - layer_norm=False, + layer_norm=False ): super(RGCNLayer, self).__init__() self.model = RelGraphConv( - in_feat=input_size, - out_feat=output_size, - num_rels=num_rels, - regularizer=regularizer, - num_bases=num_bases, - bias=bias, - activation=activation, - self_loop=self_loop, - dropout=dropout, - layer_norm=layer_norm, - ) + in_feat=input_size, + out_feat=output_size, + num_rels=num_rels, + regularizer=regularizer, + num_bases=num_bases, + bias=bias, + activation=activation, + self_loop=self_loop, + dropout=dropout, + layer_norm=layer_norm) def forward(self, graph, feat, etypes, norm=None): return self.model(graph, feat, etypes, norm) diff --git a/examples/pytorch/rgcn_hetero/__init__.py b/examples/pytorch/rgcn_hetero/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/pytorch/rgcn_hetero/main.py b/examples/pytorch/rgcn_hetero/main.py new file mode 100644 index 00000000..511f7279 --- /dev/null +++ b/examples/pytorch/rgcn_hetero/main.py @@ -0,0 +1,132 @@ +"""Deprecated RGCN code on Heterogeneous graph due to +the lack of support from data structure. The following supports +are needed (but not limit to): +- Redefine the feature data structure of node/edge + - Index node/edge ids by their type. + - Enable type indexed features. +- Make corresponding changes on views. +- Make corresponding changes on set/get features functions. + +This example bypasses it by storing the features in the model +itself. It is a code trick and therefore not recommended to +the user. +""" +import argparse +import torch +import dgl +import torch.nn.functional as F +from torchmetrics.functional import accuracy +from rgcn_hetero import RGCNHetero +from dgl.data.rdf import AIFBDataset, MUTAGDataset, BGSDataset, AMDataset +from graph4nlp.pytorch.data.data import from_dgl + +# Load dataset +# Reference: dgl/examples/pytorch/rgcn/entity_utils.py (https://github.com/dmlc/dgl/blob/master/examples/pytorch/rgcn/entity_utils.py) +def load_data(data_name='aifb', get_norm=False): + if data_name == 'aifb': + dataset = AIFBDataset() + elif data_name == 'mutag': + dataset = MUTAGDataset() + elif data_name == 'bgs': + dataset = BGSDataset() + else: + dataset = AMDataset() + # Load hetero-graph + hg = dataset[0] + + num_rels = len(hg.canonical_etypes) + category = dataset.predict_category + num_classes = dataset.num_classes + labels = hg.nodes[category].data.pop('labels') + train_mask = hg.nodes[category].data.pop('train_mask') + test_mask = hg.nodes[category].data.pop('test_mask') + train_idx = torch.nonzero(train_mask, as_tuple=False).squeeze() + test_idx = torch.nonzero(test_mask, as_tuple=False).squeeze() + + if get_norm: + # Calculate normalization weight for each edge, + # 1. / d, d is the degree of the destination node + for cetype in hg.canonical_etypes: + hg.edges[cetype].data['norm'] = dgl.norm_by_dst(hg, cetype).unsqueeze(1) + edata = ['norm'] + else: + edata = None + category_id = hg.ntypes.index(category) + hg.ndata.pop('label') + # g = dgl.to_homogeneous(hg, edata=edata) + g = hg + node_ids = torch.arange(g.num_nodes()) + + # find out the target node ids in g + loc = (g.ndata['_TYPE'] == category_id) + target_idx = node_ids[loc] + + return g, category, num_rels, num_classes, labels, train_idx, test_idx, target_idx + + +def main(args): + g, category, num_rels, num_classes, labels, train_idx, test_idx, target_idx = load_data(data_name=args.dataset, get_norm=True) + + graph = from_dgl(g, is_hetero=True) + num_nodes = graph.get_node_num() + model = RGCNHetero(num_hidden_layers=args.num_hidden_layers, + input_size=num_nodes, + hidden_size=args.hidden_size, + output_size=num_classes, + rel_names=list(set(g.etypes)), + node_types=list(graph.ntypes), + num_nodes={nt: len(g.ndata['_ID'][nt]) for nt in graph.ntypes}, + num_bases=args.num_bases, + use_self_loop=args.use_self_loop, + gpu=args.gpu, + dropout = args.dropout) + optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) + print("start training...") + model.train() + for epoch in range(args.num_epochs): + logits = model(graph)[category] + loss = F.cross_entropy(logits[train_idx], labels[train_idx]) + optimizer.zero_grad() + loss.backward() + optimizer.step() + + train_acc = accuracy(logits[train_idx].argmax(dim=1), labels[train_idx]).item() + print("Epoch {:05d} | Train Accuracy: {:.4f} | Train Loss: {:.4f}".format(epoch, train_acc, loss.item())) + print() + # Save Model + # torch.save(model.state_dict(), "./rgcn_model.pt") + print("start evaluating...") + model.eval() + with torch.no_grad(): + logits = model(graph)[category] + test_acc = accuracy(logits[test_idx].argmax(dim=1), labels[test_idx]).item() + print("Test Accuracy: {:.4f}".format(test_acc)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='RGCN for entity classification') + parser.add_argument("--num-hidden-layers", type=int, default=1, + help="number of hidden layers beside input/output layer") + parser.add_argument("--hidden-size", type=int, default=16, + help="dimension of hidden layer") + parser.add_argument("--gpu", type=int, default=-1, + help="GPU device number, -1 for cpu") + parser.add_argument("--num-bases", type=int, default=-1, + help="number of filter weight matrices, default: -1 [use all]") + parser.add_argument("-d", "--dataset", type=str, default='aifb', + choices=['aifb', 'mutag', 'bgs', 'am'], + help="dataset to use") + parser.add_argument("--use-self-loop", type=bool, default=False, + help="Consider self-loop edges or not") + parser.add_argument("--dropout", type=float, default=0.0, + help="Dropout rate") + parser.add_argument("--lr", type=float, default=1e-2, + help="Start learning rate") + parser.add_argument("--wd", type=float, default=5e-4, + help="weight decay") + parser.add_argument("--num-epochs", type=int, default=50, + help="Number of training epochs") + + args = parser.parse_args() + print(args) + main(args) \ No newline at end of file diff --git a/examples/pytorch/rgcn_hetero/rgcn_hetero.py b/examples/pytorch/rgcn_hetero/rgcn_hetero.py new file mode 100644 index 00000000..3e2a1f28 --- /dev/null +++ b/examples/pytorch/rgcn_hetero/rgcn_hetero.py @@ -0,0 +1,221 @@ +from re import S +import warnings +import dgl.function as fn +import torch +import torch.nn as nn +import torch.nn.functional as F +import dgl +from dgl.utils import check_eq_shape, expand_as_pair +from dgl.nn.pytorch import HeteroGraphConv, GraphConv + +from graph4nlp.pytorch.modules.graph_embedding_learning.base import GNNBase, GNNLayerBase + + +class RGCNHetero(GNNBase): + r"""Multi-layered `RGCN Network `__ + + .. math:: + TODO:Add Calculation. + + Parameters + ---------- + num_layers: int + Number of RGCN layers. + input_size : int, or pair of ints + Input feature size. + hidden_size: int + Hidden layer size. + output_size : int + Output feature size. + rels_names : List[str] + Names of relation(edge) types. + node_types: List[str] + Names of node types. + num_nodes: int, + Number of nodes. + num_bases : int, optional + Number of bases. Needed when ``regularizer`` is specified. Default: ``-1``. + use_self_loop : bool, optional + True to include self loop message. Default: ``False``. + gpu: int, optional + GPU device number. Default: ``-1`` (CPU) + dropout : float, optional + Dropout rate. Default: ``0.0`` + """ + + def __init__( + self, + num_hidden_layers, + input_size, + hidden_size, + output_size, + rel_names, + node_types, + num_nodes, + num_bases=-1, + use_self_loop=False, + gpu=-1, + dropout=0.0 + ): + super(RGCNHetero, self).__init__() + self.num_hidden_layers = num_hidden_layers + self.rel_names = rel_names + self.num_bases = num_bases + self.num_nodes = num_nodes + self.use_self_loop = use_self_loop + self.dropout = dropout + self.gpu = gpu + + self.embs = nn.ParameterDict({}) + for nt in node_types: + embed = nn.Parameter(torch.Tensor(num_nodes[nt], hidden_size)) + nn.init.xavier_uniform_(embed, gain=nn.init.calculate_gain('relu')) + self.embs[nt] = embed + + self.RGCN_layers = nn.ModuleList() + + # hidden layers + for l in range(self.num_hidden_layers): + # due to multi-head, the input_size = hidden_size * num_heads + self.RGCN_layers.append( + RGCNLayerHetero( + hidden_size, + hidden_size, + rel_names=rel_names, + num_bases=self.num_bases, + activation=F.relu, + self_loop=self.use_self_loop, + dropout=self.dropout + ) + ) + # output projection + self.RGCN_layers.append( + RGCNLayerHetero( + hidden_size, + output_size, + rel_names=self.rel_names, + num_bases=self.num_bases, + activation=F.relu, + self_loop=self.use_self_loop, + dropout=self.dropout + ) + ) + self.h = self.embs + if self.gpu != -1: + self.to(device=self.gpu) + + def forward(self, graph, h=None): + r"""Compute RGCN layer. + + Parameters + ---------- + graph : GraphData + The graph with node feature stored in the feature field named as + "node_feat". + The node features are used for message passing. + + Returns + ------- + graph : GraphData + The graph with generated node embedding stored in the feature field + named as "node_emb". + """ + + + # transfer the current NLPgraph to DGL graph + g = graph.to_dgl() + if h is None: + h = self.embs + for l in range(self.num_hidden_layers): + h = self.RGCN_layers[l](g, h) + logits = self.RGCN_layers[-1](g, h) + + # graph.node_features['node_feat'] = h {'type1': (num_node_type1 x emb_dim), 'type2': (num_node_type2 x emb_dim)} + # graph.node_features["node_emb"] = logits # put the results into the NLPGraph + return logits + + +class RGCNLayerHetero(GNNLayerBase): + r"""A wrapper for RelGraphConv in DGL. + + .. math:: + TODO + + Parameters + ---------- + input_size : int, or pair of ints + Input feature size. + output_size : int + Output feature size. + num_rels: int + number of relations + regularizer : str, optional + Which weight regularizer to use "basis" or "bdd": + - "basis" is short for basis-decomposition. + - "bdd" is short for block-diagonal-decomposition. + Default applies no regularization. + num_bases : int, optional + Number of bases. Needed when ``regularizer`` is specified. Default: ``None``. + bias : bool, optional + True if bias is added. Default: ``True``. + activation : callable, optional + Activation function. Default: ``None``. + self_loop : bool, optional + True to include self loop message. Default: ``True``. + dropout : float, optional + Dropout rate. Default: ``0.0`` + layer_norm: float, optional + Add layer norm. Default: ``False`` + """ + + def __init__( + self, + input_size, + output_size, + rel_names, + num_bases=None, + activation=None, + self_loop=False, + dropout=0.0 + ): + super(RGCNLayerHetero, self).__init__() + self.input_size = input_size + self.output_size = output_size + self.rel_names = rel_names + self.num_bases = num_bases + # self.bias = bias + self.activation = activation + self.self_loop = self_loop + self.dropout = dropout + + self.conv = HeteroGraphConv( + { + rel: GraphConv(input_size, output_size, norm='right', weight=False, bias=False) for rel in rel_names + } + ) + self.dropout = nn.Dropout(dropout) + + + def forward(self, graph, inputs): + """ + + Parameters: + ---------- + graph: DGLHeteroGraph + The graph + inputs: dict[str, torch.Tensor] + New node features for each node type + """ + graph = graph.local_var() + inputs_src = inputs_dst = inputs + + hs = self.conv(graph, inputs) + + def _apply(ntype, h): + if self.self_loop: + h = h + torch.matmul(inputs_dst[ntype], self.loop_weight) + if self.activation: + h = self.activation(h) + return self.dropout(h) + + return {ntype: _apply(ntype, h) for ntype, h in hs.items()} diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/__init__.py b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_amr_undirected.json b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_amr_undirected.json new file mode 100644 index 00000000..7cd368b7 --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_amr_undirected.json @@ -0,0 +1,14 @@ +{ + "config_path": "examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml", + "checkpoint_args.checkpoint_name": "node_emb_sage_undirected.pt", + "checkpoint_args.out_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/save_for_amr", + "env_args.gpuid": 1, + "training_args.batch_size": 20, + "training_args.max_epochs": 150, + "model_args.graph_construction_args.graph_construction_share.root_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data", + "inference_args.inference_data_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data_inference", + "model_args.graph_construction_name": "amr", + "model_args.graph_construction_args.graph_construction_share.topology_subdir": "AMRGraphForRGCN", + "model_args.graph_initialization_args.embedding_style.single_token_item": false, + "model_args.graph_initialization_args.embedding_style.emb_strategy": "w2v_bilstm_amr_pos" +} \ No newline at end of file diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json new file mode 100644 index 00000000..95451cef --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json @@ -0,0 +1,10 @@ +{ + "config_path": "examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml", + "checkpoint_args.checkpoint_name": "node_emb_sage_undirected.pt", + "checkpoint_args.out_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/save_for_amr", + "env_args.gpuid": 1, + "training_args.batch_size": 20, + "training_args.max_epochs": 150, + "model_args.graph_construction_args.graph_construction_share.root_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data", + "inference_args.inference_data_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data_inference" +} \ No newline at end of file diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml new file mode 100644 index 00000000..867a50f3 --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_amr.yaml @@ -0,0 +1,130 @@ +# Users can import user customized yaml files or library provided default yaml files +includes: [] + # - $/configs/defaults.yaml + # In the above example path, '$/' indicates the library directory which contains the config folder + + +preprocessing_args: + min_freq: 1 + pretrained_word_emb_name: null + + +model_args: + graph_construction_name: "dependency" + graph_initialization_name: "defaults" + graph_embedding_name: "rgcn" + decoder_name: "stdtree" + + + graph_construction_args: + graph_construction_share: + root_dir: 'examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data' + topology_subdir: 'DependencyGraphForRGCN' + thread_number: 15 + share_vocab: True + port: 9000 + timeout: 15000 + + nlp_processor_args: + name: "stanza" + args: + annotators: ["tokenize", "ssplit", "pos", "ner"] + corenlp_dir: "./corenlp" + endpoint: "http://localhost:9002" + memory: "4G" + properties: + tokenize.options: + splitHyphenated: False + normalizeParentheses: False + normalizeOtherBrackets: False + tokenize.whitespace: True + ssplit.isOneSentence: True + + graph_construction_private: + edge_strategy: 'heterogeneous' + merge_strategy: 'tailhead' + sequential_link: true + as_node: false + sim_metric_type: 'weighted_cosine' + num_heads: 1 + top_k_neigh: null + epsilon_neigh: 0.5 + smoothness_ratio: 0.1 + connectivity_ratio: 0.05 + sparsity_ratio: 0.1 + + graph_initialization_args: + input_size: 300 + hidden_size: 300 + word_dropout: 0.1 + rnn_dropout: 0.1 + # word_dropout: 0.2 + # rnn_dropout: 0.3 + fix_bert_emb: false + fix_word_emb: false + embedding_style: + single_token_item: true + emb_strategy: "w2v_bilstm" + num_rnn_layers: 1 + bert_model_name: null + bert_lower_case: null + + graph_embedding_args: + graph_embedding_share: + num_layers: 1 + input_size: 300 + hidden_size: 300 + output_size: 300 + direction_option: "undirected" + feat_drop: 0.0 + attn_drop: 0.0 + graph_embedding_private: + aggregator_type: "lstm" + bias: true + norm: null + activation: "relu" + use_edge_weight: true + + decoder_args: + rnn_decoder_share: + rnn_type: "lstm" + input_size: 300 + hidden_size: 300 + rnn_emb_input_size: 300 + use_copy: true + graph_pooling_strategy: null + attention_type: "uniform" + fuse_strategy: "concatenate" + dropout: 0.1 + teacher_forcing_rate: 1.0 + rnn_decoder_private: + max_decoder_step: 50 + max_tree_depth: 50 + use_sibling: false + +training_args: + learning_rate: 0.001 + init_weight: 0.08 + weight_decay: 0 + max_epochs: 150 + grad_clip: 5 + batch_size: 20 + + +inference_args: + beam_size: 4 + inference_data_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data_inference" + + +evaluation_args: + # Metrics for evaluation + + +checkpoint_args: + out_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/save" + checkpoint_name: "best.pt" + + +env_args: + seed: 0 + gpuid: -1 diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml new file mode 100644 index 00000000..52341500 --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml @@ -0,0 +1,130 @@ +# Users can import user customized yaml files or library provided default yaml files +includes: [] + # - $/configs/defaults.yaml + # In the above example path, '$/' indicates the library directory which contains the config folder + + +preprocessing_args: + min_freq: 1 + pretrained_word_emb_name: null + + +model_args: + graph_construction_name: "dependency" + graph_initialization_name: "defaults" + graph_embedding_name: "rgcn" + decoder_name: "stdtree" + + + graph_construction_args: + graph_construction_share: + root_dir: 'examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data' + topology_subdir: 'DependencyGraphForRGCN' + thread_number: 15 + share_vocab: True + port: 9000 + timeout: 15000 + + nlp_processor_args: + name: "stanza" + args: + annotators: ["tokenize", "ssplit", "pos", "depparse"] + corenlp_dir: "./corenlp" + endpoint: "http://localhost:9002" + memory: "4G" + properties: + tokenize.options: + splitHyphenated: False + normalizeParentheses: False + normalizeOtherBrackets: False + tokenize.whitespace: True + ssplit.isOneSentence: False + + graph_construction_private: + edge_strategy: 'heterogeneous' + merge_strategy: 'tailhead' + sequential_link: true + as_node: false + sim_metric_type: 'weighted_cosine' + num_heads: 1 + top_k_neigh: null + epsilon_neigh: 0.5 + smoothness_ratio: 0.1 + connectivity_ratio: 0.05 + sparsity_ratio: 0.1 + + graph_initialization_args: + input_size: 300 + hidden_size: 300 + word_dropout: 0.1 + rnn_dropout: 0.1 + # word_dropout: 0.2 + # rnn_dropout: 0.3 + fix_bert_emb: false + fix_word_emb: false + embedding_style: + single_token_item: true + emb_strategy: "w2v_bilstm" + num_rnn_layers: 1 + bert_model_name: null + bert_lower_case: null + + graph_embedding_args: + graph_embedding_share: + num_layers: 1 + input_size: 300 + hidden_size: 300 + output_size: 300 + direction_option: "undirected" + feat_drop: 0.0 + attn_drop: 0.0 + graph_embedding_private: + aggregator_type: "lstm" + bias: true + norm: null + activation: "relu" + use_edge_weight: true + + decoder_args: + rnn_decoder_share: + rnn_type: "lstm" + input_size: 300 + hidden_size: 300 + rnn_emb_input_size: 300 + use_copy: true + graph_pooling_strategy: null + attention_type: "uniform" + fuse_strategy: "concatenate" + dropout: 0.1 + teacher_forcing_rate: 1.0 + rnn_decoder_private: + max_decoder_step: 50 + max_tree_depth: 50 + use_sibling: false + +training_args: + learning_rate: 0.001 + init_weight: 0.08 + weight_decay: 0 + max_epochs: 150 + grad_clip: 5 + batch_size: 20 + + +inference_args: + beam_size: 4 + inference_data_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data_inference" + + +evaluation_args: + # Metrics for evaluation + + +checkpoint_args: + out_dir: "examples/pytorch/semantic_parsing/graph2tree/jobs/save" + checkpoint_name: "best.pt" + + +env_args: + seed: 0 + gpuid: -1 diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py new file mode 100644 index 00000000..0bbfd62f --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py @@ -0,0 +1,106 @@ +""" + The inference code. + In this file, we will run the inference by using the prediction API \ + in the GeneratorInferenceWrapper. + The GeneratorInferenceWrapper takes the raw inputs and produce the outputs. +""" +import argparse +import random +import warnings +import numpy as np +import torch +from utils import AMRDataItem + +from graph4nlp.pytorch.datasets.mawps import MawpsDatasetForTree, tokenize_mawps +from graph4nlp.pytorch.inference_wrapper.generator_inference_wrapper_for_tree import ( + GeneratorInferenceWrapper, +) +from graph4nlp.pytorch.modules.utils.config_utils import load_json_config +from examples.pytorch.amr_graph_construction.amr_graph_construction import AMRGraphConstruction +from utils import AMRDataItem, RGCNGraph2Tree, InferenceText2TreeDataset +warnings.filterwarnings("ignore") + + +class Mawps: + def __init__(self, opt=None): + super(Mawps, self).__init__() + self.opt = opt + + seed = self.opt["env_args"]["seed"] + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if self.opt["env_args"]["gpuid"] == -1: + self.device = torch.device("cpu") + else: + self.device = torch.device("cuda:{}".format(self.opt["env_args"]["gpuid"])) + + self._build_model() + + def _build_model(self): + self.model = RGCNGraph2Tree.load_checkpoint( + self.opt["checkpoint_args"]["out_dir"], self.opt["checkpoint_args"]["checkpoint_name"] + ).to(self.device) + + self.inference_tool = GeneratorInferenceWrapper( + cfg=self.opt, + model=self.model, + beam_size=2, + lower_case=True, + tokenizer=tokenize_mawps, + dataset=InferenceText2TreeDataset, + data_item=AMRDataItem, + topology_builder=(AMRGraphConstruction if self.model.graph_construction_name == "amr" else None) + ) + + @torch.no_grad() + def translate(self): + self.model.eval() + ret = self.inference_tool.predict( + raw_contents=[ + "2 dogs are barking . 1 more dogs start to bark . how many dogs are barking" + ], + batch_size=1, + ) + print(ret) + + +################################################################################ +# ArgParse and Helper Functions # +################################################################################ +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-json_config", + "--json_config", + required=True, + type=str, + help="path to the json config file", + ) + args = vars(parser.parse_args()) + + return args + + +def print_config(config): + import pprint + + print("**************** MODEL CONFIGURATION ****************") + pprint.pprint(config) + print("**************** MODEL CONFIGURATION ****************") + + +if __name__ == "__main__": + import platform + import multiprocessing + + #if platform.system() == "Darwin": + multiprocessing.set_start_method("spawn") + + cfg = get_args() + config = load_json_config(cfg["json_config"]) + # print_config(config) + + runner = Mawps(opt=config) + runner.translate() diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py new file mode 100644 index 00000000..81c5e2b4 --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py @@ -0,0 +1,286 @@ +import numpy as np +import torch +import torch.optim as optim +from torch.utils.data import DataLoader +from tqdm import tqdm + +from graph4nlp.pytorch.data.dataset import Text2TreeDataItem +from graph4nlp.pytorch.datasets.jobs import JobsDatasetForTree +from graph4nlp.pytorch.models.graph2tree import Graph2Tree +from graph4nlp.pytorch.modules.utils.config_utils import load_json_config +from graph4nlp.pytorch.modules.utils.tree_utils import Tree + +import argparse +import copy +import random +import time +import warnings +from examples.pytorch.amr_graph_construction.amr_graph_construction import AMRGraphConstruction +from utils import AMRDataItem, AMRGraph2Tree, EdgeText2TreeDataset, RGCNGraph2Tree + +warnings.filterwarnings("ignore") + + +class Jobs: + def __init__(self, opt=None): + super(Jobs, self).__init__() + self.opt = opt + + seed = self.opt["env_args"]["seed"] + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + if self.opt["env_args"]["gpuid"] == -1: + self.device = torch.device("cpu") + else: + self.device = torch.device("cuda:{}".format(self.opt["env_args"]["gpuid"])) + + self.use_copy = self.opt["model_args"]["decoder_args"]["rnn_decoder_share"]["use_copy"] + self.use_share_vocab = self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["share_vocab"] + self.data_dir = self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["root_dir"] + + self._build_dataloader() + self._build_model() + self._build_optimizer() + + def _build_dataloader(self): + graph_type = self.opt["model_args"]["graph_construction_name"] + para_dic = { + "root_dir": self.data_dir, + "word_emb_size": self.opt["model_args"]["graph_initialization_args"]["input_size"], + "topology_subdir": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["topology_subdir"], + "edge_strategy": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_private" + ]["edge_strategy"], + "graph_construction_name": self.opt["model_args"]["graph_construction_name"], + "share_vocab": self.use_share_vocab, + "enc_emb_size": self.opt["model_args"]["graph_initialization_args"]["input_size"], + "dec_emb_size": self.opt["model_args"]["decoder_args"]["rnn_decoder_share"][ + "input_size" + ], + "dynamic_init_graph_name": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_private" + ].get("dynamic_init_graph_name", None), + "min_word_vocab_freq": self.opt["preprocessing_args"]["min_freq"], + "pretrained_word_emb_name": self.opt["preprocessing_args"]["pretrained_word_emb_name"], + "nlp_processor_args": self.opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ]["nlp_processor_args"], + "dataitem": Text2TreeDataItem if graph_type != "amr" else AMRDataItem, + #"dataitem": AMRDataItem, + "topology_builder": AMRGraphConstruction if graph_type == "amr" else None, + } + + dataset = EdgeText2TreeDataset(**para_dic) + + self.train_data_loader = DataLoader( + dataset.train, + batch_size=self.opt["training_args"]["batch_size"], + shuffle=True, + num_workers=0, + collate_fn=dataset.collate_fn, + ) + self.test_data_loader = DataLoader( + dataset.test, batch_size=1, shuffle=False, num_workers=0, collate_fn=dataset.collate_fn + ) + self.vocab_model = dataset.vocab_model + self.src_vocab = self.vocab_model.in_word_vocab + self.tgt_vocab = self.vocab_model.out_word_vocab + #self.num_rel = len(dataset.edge_vocab) + #print(dataset.edge_vocab) + self.share_vocab = self.vocab_model.share_vocab if self.use_share_vocab else None + + def _build_model(self): + """For encoder-decoder""" + print(self.opt["model_args"]["graph_embedding_name"]) + if self.opt["model_args"]["graph_embedding_name"] == "rgcn": + if self.opt["model_args"]["graph_construction_name"] == "amr": + self.model = AMRGraph2Tree.from_args(opt=self.opt, vocab_model=self.vocab_model) + else: + self.model = RGCNGraph2Tree.from_args(opt=self.opt, vocab_model=self.vocab_model) + else: + self.model = Graph2Tree.from_args(self.opt, vocab_model=self.vocab_model) + self.model.init(self.opt["training_args"]["init_weight"]) + self.model.to(self.device) + + def _build_optimizer(self): + optim_state = { + "learningRate": self.opt["training_args"]["learning_rate"], + "weight_decay": self.opt["training_args"]["weight_decay"], + } + parameters = [p for p in self.model.parameters() if p.requires_grad] + self.optimizer = optim.Adam( + parameters, lr=optim_state["learningRate"], weight_decay=optim_state["weight_decay"] + ) + + def prepare_ext_vocab(self, batch_graph, src_vocab): + oov_dict = copy.deepcopy(src_vocab) + token_matrix = [] + for n in batch_graph.node_attributes: + node_token = n["token"] + if (n.get("type") is None or n.get("type") == 0) and oov_dict.get_symbol_idx( + node_token + ) == oov_dict.get_symbol_idx(oov_dict.unk_token): + oov_dict.add_symbol(node_token) + token_matrix.append(oov_dict.get_symbol_idx(node_token)) + batch_graph.node_features["token_id_oov"] = torch.tensor(token_matrix, dtype=torch.long).to( + self.device + ) + return oov_dict + + def train_epoch(self, epoch): + loss_to_print = 0 + num_batch = len(self.train_data_loader) + for _, data in tqdm( + enumerate(self.train_data_loader), + desc=f"Epoch {epoch:02d}", + total=len(self.train_data_loader), + ): + batch_graph, batch_tree_list, batch_original_tree_list = ( + data["graph_data"], + data["dec_tree_batch"], + data["original_dec_tree_batch"], + ) + batch_graph = batch_graph.to(self.device) + self.optimizer.zero_grad() + oov_dict = ( + self.prepare_ext_vocab(batch_graph, self.src_vocab) if self.use_copy else None + ) + + if self.use_copy: + batch_tree_list_refined = [] + for item in batch_original_tree_list: + tgt_list = oov_dict.get_symbol_idx_for_list(item.strip().split()) + tgt_tree = Tree.convert_to_tree(tgt_list, 0, len(tgt_list), oov_dict) + batch_tree_list_refined.append(tgt_tree) + loss = self.model( + batch_graph, + batch_tree_list_refined if self.use_copy else batch_tree_list, + oov_dict=oov_dict, + ) + loss.backward() + torch.nn.utils.clip_grad_value_( + self.model.parameters(), self.opt["training_args"]["grad_clip"] + ) + self.optimizer.step() + loss_to_print += loss + print("-------------\nLoss = {:.3f}".format(loss_to_print / num_batch)) + + def train(self): + print("-------------\nStarting training.") + best_acc = 0.0 + best_model = None + for epoch in range(1, self.opt["training_args"]["max_epochs"] + 1): + self.model.train() + self.train_epoch(epoch) + if epoch >= 5 and epoch % 10 == 0: + val_acc = self.eval(self.model) + if val_acc > best_acc: + best_acc = val_acc + best_model = self.model + best_model.save_checkpoint( + self.opt["checkpoint_args"]["out_dir"], self.opt["checkpoint_args"]["checkpoint_name"] + ) + print(f"Best Accuracy: {best_acc:.4f}") + + def eval(self, model): + from examples.pytorch.semantic_parsing.graph2tree.jobs.src.evaluation import compute_tree_accuracy + + model.eval() + reference_list = [] + candidate_list = [] + for data in tqdm(self.test_data_loader, desc="Eval: "): + eval_input_graph, _, batch_original_tree_list = ( + data["graph_data"], + data["dec_tree_batch"], + data["original_dec_tree_batch"], + ) + eval_input_graph = eval_input_graph.to(self.device) + oov_dict = self.prepare_ext_vocab(eval_input_graph, self.src_vocab) + + if self.use_copy: + assert len(batch_original_tree_list) == 1 + reference = oov_dict.get_symbol_idx_for_list(batch_original_tree_list[0].split()) + eval_vocab = oov_dict + else: + assert len(batch_original_tree_list) == 1 + reference = model.tgt_vocab.get_symbol_idx_for_list( + batch_original_tree_list[0].split() + ) + eval_vocab = self.tgt_vocab + candidate = model.translate( + eval_input_graph, + oov_dict=oov_dict, + use_beam_search=True, + beam_size=self.opt["inference_args"]["beam_size"], + ) + candidate = [int(c) for c in candidate] + num_left_paren = sum(1 for c in candidate if eval_vocab.idx2symbol[int(c)] == "(") + num_right_paren = sum(1 for c in candidate if eval_vocab.idx2symbol[int(c)] == ")") + diff = num_left_paren - num_right_paren + if diff > 0: + for _ in range(diff): + candidate.append(self.test_data_loader.tgt_vocab.symbol2idx[")"]) + elif diff < 0: + candidate = candidate[:diff] + # ref_str = convert_to_string(reference, eval_vocab) + # cand_str = convert_to_string(candidate, eval_vocab) + + reference_list.append(reference) + candidate_list.append(candidate) + eval_acc = compute_tree_accuracy(candidate_list, reference_list, eval_vocab) + print(f"Accuracy: {eval_acc:.4f}\n") + return eval_acc + + +################################################################################ +# ArgParse and Helper Functions # +################################################################################ +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-json_config", + "--json_config", + required=True, + type=str, + help="path to the json config file", + ) + args = vars(parser.parse_args()) + + return args + + +def print_config(config): + import pprint + + print("**************** MODEL CONFIGURATION ****************") + pprint.pprint(config) + print("**************** MODEL CONFIGURATION ****************") + + +if __name__ == "__main__": + torch.multiprocessing.set_start_method('spawn') + import platform + import multiprocessing + + if platform.system() == "Darwin": + multiprocessing.set_start_method("spawn") + + cfg = get_args() + config = load_json_config(cfg["json_config"]) + print_config(config) + + start = time.time() + runner = Jobs(opt=config) + + runner.train() + + end = time.time() + print("total time: {} minutes\n".format((end - start) / 60)) diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/utils.py b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/utils.py new file mode 100644 index 00000000..76d3406a --- /dev/null +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/utils.py @@ -0,0 +1,1125 @@ +from abc import abstractmethod +import copy +import warnings +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from copy import deepcopy +from graph4nlp.pytorch.data.data import GraphData, from_batch +from graph4nlp.pytorch.data.dataset import DataItem, Text2TreeDataset + +from graph4nlp.pytorch.datasets.jobs import JobsDatasetForTree +from graph4nlp.pytorch.models.graph2tree import Graph2Tree +from graph4nlp.pytorch.modules.graph_embedding_initialization.embedding_construction import BertEmbedding, EmbeddingConstruction, MeanEmbedding, RNNEmbedding, WordEmbedding +from graph4nlp.pytorch.modules.utils.generic_utils import dropout_fn +from graph4nlp.pytorch.modules.utils.tree_utils import Tree + +from examples.pytorch.rgcn.rgcn import RGCN +from graph4nlp.pytorch.modules.utils.vocab_utils import Vocab + +warnings.filterwarnings("ignore") + +class AmrEmbeddingConstruction(EmbeddingConstruction): + """Initial graph embedding construction class. + + Parameters + ---------- + word_vocab : Vocab + The word vocabulary. + single_token_item : bool + Specify whether the item (i.e., node or edge) contains single token or multiple tokens. + emb_strategy : str + Specify the embedding construction strategy including the following options: + - 'w2v': use word2vec embeddings. + - 'w2v_bilstm': use word2vec embeddings, and apply BiLSTM encoders. + - 'w2v_bigru': use word2vec embeddings, and apply BiGRU encoders. + - 'bert': use BERT embeddings. + - 'bert_bilstm': use BERT embeddings, and apply BiLSTM encoders. + - 'bert_bigru': use BERT embeddings, and apply BiGRU encoders. + - 'w2v_bert': use word2vec and BERT embeddings. + - 'w2v_bert_bilstm': use word2vec and BERT embeddings, and apply BiLSTM encoders. + - 'w2v_bert_bigru': use word2vec and BERT embeddings, and apply BiGRU encoders. + Note that if 'w2v' is not applied, `pretrained_word_emb_name` specified in Dataset APIs + will be superseded. + hidden_size : int, optional + The hidden size of RNN layer, default: ``None``. + num_rnn_layers : int, optional + The number of RNN layers, default: ``1``. + fix_word_emb : boolean, optional + Specify whether to fix pretrained word embeddings, default: ``True``. + fix_bert_emb : boolean, optional + Specify whether to fix pretrained BERT embeddings, default: ``True``. + bert_model_name : str, optional + Specify the BERT model name, default: ``'bert-base-uncased'``. + bert_lower_case : bool, optional + Specify whether to lower case the input text for BERT embeddings, default: ``True``. + word_dropout : float, optional + Dropout ratio for word embedding, default: ``None``. + rnn_dropout : float, optional + Dropout ratio for RNN embedding, default: ``None``. + + Note + ---------- + word_emb_type : str or list of str + Specify pretrained word embedding types including "w2v", "node_edge_bert", + or "seq_bert". + node_edge_emb_strategy : str + Specify node/edge embedding strategies including "mean", "bilstm" and "bigru". + seq_info_encode_strategy : str + Specify strategies of encoding sequential information in raw text + data including "none", "bilstm" and "bigru". You might + want to do this in some situations, e.g., when all the nodes are single + tokens extracted from the raw text. + + 1) single-token node (i.e., single_token_item=`True`): + a) 'w2v', 'bert', 'w2v_bert' + b) node_edge_emb_strategy: 'mean' + c) seq_info_encode_strategy: 'none', 'bilstm', 'bigru' + emb_strategy: 'w2v', 'w2v_bilstm', 'w2v_bigru', + 'bert', 'bert_bilstm', 'bert_bigru', + 'w2v_bert', 'w2v_bert_bilstm', 'w2v_bert_bigru' + + 2) multi-token node (i.e., single_token_item=`False`): + a) 'w2v', 'bert', 'w2v_bert' + b) node_edge_emb_strategy: 'mean', 'bilstm', 'bigru' + c) seq_info_encode_strategy: 'none' + emb_strategy: ('w2v', 'w2v_bilstm', 'w2v_bigru', + 'bert', 'bert_bilstm', 'bert_bigru', + 'w2v_bert', 'w2v_bert_bilstm', 'w2v_bert_bigru') + """ + + def __init__( + self, + word_vocab, + single_token_item, + emb_strategy="w2v_bilstm", + hidden_size=None, + num_rnn_layers=1, + fix_word_emb=True, + fix_bert_emb=True, + bert_model_name="bert-base-uncased", + bert_lower_case=True, + word_dropout=None, + bert_dropout=None, + rnn_dropout=None, + ): + super(EmbeddingConstruction, self).__init__() + self.word_dropout = word_dropout + self.bert_dropout = bert_dropout + self.rnn_dropout = rnn_dropout + self.single_token_item = single_token_item + + assert emb_strategy in ( + "w2v", + "w2v_bilstm", + "w2v_bigru", + "bert", + "bert_bilstm", + "bert_bigru", + "w2v_bert", + "w2v_bert_bilstm", + "w2v_bert_bigru", + "w2v_amr", + "w2v_bilstm_amr", + "w2v_bilstm_amr_pos", + ), "emb_strategy must be one of ('w2v', 'w2v_bilstm', 'w2v_bigru', 'bert', 'bert_bilstm', " + "'bert_bigru', 'w2v_bert', 'w2v_bert_bilstm', 'w2v_bert_bigru')" + + word_emb_type = set() + if single_token_item: + node_edge_emb_strategy = None + if "w2v" in emb_strategy: + word_emb_type.add("w2v") + + if "bert" in emb_strategy: + word_emb_type.add("seq_bert") + + if "bilstm" in emb_strategy: + seq_info_encode_strategy = "bilstm" + elif "bigru" in emb_strategy: + seq_info_encode_strategy = "bigru" + else: + seq_info_encode_strategy = "none" + else: + seq_info_encode_strategy = "none" + if "amr" in emb_strategy: + seq_info_encode_strategy = "bilstm" + + if "pos" in emb_strategy: + word_emb_type.add("pos") + #word_emb_type.add("entity_label") + word_emb_type.add("position") + + if "w2v" in emb_strategy: + word_emb_type.add("w2v") + + if "bert" in emb_strategy: + word_emb_type.add("node_edge_bert") + + if "bilstm" in emb_strategy: + node_edge_emb_strategy = "bilstm" + elif "bigru" in emb_strategy: + node_edge_emb_strategy = "bigru" + else: + node_edge_emb_strategy = "mean" + + word_emb_size = 0 + self.word_emb_layers = nn.ModuleDict() + if "w2v" in word_emb_type: + self.word_emb_layers["w2v"] = WordEmbedding( + word_vocab.embeddings.shape[0], + word_vocab.embeddings.shape[1], + pretrained_word_emb=word_vocab.embeddings, + fix_emb=fix_word_emb, + ) + word_emb_size += word_vocab.embeddings.shape[1] + + if "node_edge_bert" in word_emb_type: + self.word_emb_layers["node_edge_bert"] = BertEmbedding( + name=bert_model_name, fix_emb=fix_bert_emb, lower_case=bert_lower_case + ) + word_emb_size += self.word_emb_layers["node_edge_bert"].bert_model.config.hidden_size + + if "seq_bert" in word_emb_type: + self.word_emb_layers["seq_bert"] = BertEmbedding( + name=bert_model_name, fix_emb=fix_bert_emb, lower_case=bert_lower_case + ) + + if node_edge_emb_strategy in ("bilstm", "bigru"): + self.node_edge_emb_layer = RNNEmbedding( + word_emb_size, + hidden_size, + bidirectional=True, + num_layers=num_rnn_layers, + rnn_type="lstm" if node_edge_emb_strategy == "bilstm" else "gru", + dropout=rnn_dropout, + ) + rnn_input_size = hidden_size + elif node_edge_emb_strategy == "mean": + self.node_edge_emb_layer = MeanEmbedding() + rnn_input_size = word_emb_size + else: + rnn_input_size = word_emb_size + + if "pos" in word_emb_type: + self.word_emb_layers["pos"] = WordEmbedding(50, 50) + rnn_input_size += 50 + + if "entity_label" in word_emb_type: + self.word_emb_layers["entity_label"] = WordEmbedding(50, 50) + rnn_input_size += 50 + + if "position" in word_emb_type: + pass + + if "seq_bert" in word_emb_type: + rnn_input_size += self.word_emb_layers["seq_bert"].bert_model.config.hidden_size + + if seq_info_encode_strategy in ("bilstm", "bigru"): + self.output_size = hidden_size + self.seq_info_encode_layer = RNNEmbedding( + rnn_input_size, + hidden_size, + bidirectional=True, + num_layers=num_rnn_layers, + rnn_type="lstm" if seq_info_encode_strategy == "bilstm" else "gru", + dropout=rnn_dropout, + ) + + else: + self.output_size = rnn_input_size + self.seq_info_encode_layer = None + + #self.fc = nn.Linear(376, 300) + + def forward(self, batch_gd): + """Compute initial node/edge embeddings. + + Parameters + ---------- + batch_gd : GraphData + The input graph data. + + Returns + ------- + GraphData + The output graph data with updated node embeddings. + """ + feat = [] + if self.single_token_item: # single-token node graph + token_ids = batch_gd.batch_node_features["token_id"] + if "w2v" in self.word_emb_layers: + word_feat = self.word_emb_layers["w2v"](token_ids).squeeze(-2) + word_feat = dropout_fn( + word_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + feat.append(word_feat) + + else: # multi-token node graph + token_ids = batch_gd.node_features["token_id"] + if "w2v" in self.word_emb_layers: + word_feat = self.word_emb_layers["w2v"](token_ids) + word_feat = dropout_fn( + word_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + feat.append(word_feat) + if any(batch_gd.batch_graph_attributes): + tot = 0 + gd_list = from_batch(batch_gd) + for i, g in enumerate(gd_list): + sentence_id = g.graph_attributes["sentence_id"].to(batch_gd.device) + seq_feat = [] + if "w2v" in self.word_emb_layers: + word_feat = self.word_emb_layers["w2v"](sentence_id) + word_feat = dropout_fn( + word_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + seq_feat.append(word_feat) + else: + RuntimeError("No word embedding layer") + if "pos" in self.word_emb_layers: + sentence_pos = g.graph_attributes["pos_tag_id"].to(batch_gd.device) + pos_feat = self.word_emb_layers["pos"](sentence_pos) + pos_feat = dropout_fn( + pos_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + seq_feat.append(pos_feat) + + if "entity_label" in self.word_emb_layers: + sentence_entity_label = g.graph_attributes["entity_label_id"].to(batch_gd.device) + entity_label_feat = self.word_emb_layers["entity_label"](sentence_entity_label) + entity_label_feat = dropout_fn( + entity_label_feat, self.word_dropout, shared_axes=[-2], training=self.training + ) + seq_feat.append(entity_label_feat) + + seq_feat = torch.cat(seq_feat, dim=-1) + + raw_tokens = [dd.strip().split() for dd in g.graph_attributes["sentence"]] + l = [len(s) for s in raw_tokens] + rnn_state = self.seq_info_encode_layer( + seq_feat, torch.LongTensor(l).to(batch_gd.device) + ) + if isinstance(rnn_state, (tuple, list)): + rnn_state = rnn_state[0] + + # update node features + for j in range(g.get_node_num()): + id = g.node_attributes[j]["sentence_id"] + if g.node_attributes[j]["id"] in batch_gd.batch_graph_attributes[i]["mapping"][id]: + rel_list = batch_gd.batch_graph_attributes[i]["mapping"][id][g.node_attributes[j]["id"]] + state = [] + for rel in rel_list: + if rel[1] == "node": + state.append(rnn_state[id][rel[0]]) + # replace embedding of the node + if len(state) > 0: + feat[0][tot + j][0] = torch.stack(state, 0).mean(0) + + tot += g.get_node_num() + + if "node_edge_bert" in self.word_emb_layers: + input_data = [ + batch_gd.node_attributes[i]["token"].strip().split(" ") + for i in range(batch_gd.get_node_num()) + ] + node_edge_bert_feat = self.word_emb_layers["node_edge_bert"](input_data) + node_edge_bert_feat = dropout_fn( + node_edge_bert_feat, self.bert_dropout, shared_axes=[-2], training=self.training + ) + feat.append(node_edge_bert_feat) + + if len(feat) > 0: + feat = torch.cat(feat, dim=-1) + if not any(batch_gd.batch_graph_attributes): + node_token_lens = torch.clamp((token_ids != Vocab.PAD).sum(-1), min=1) + feat = self.node_edge_emb_layer(feat, node_token_lens) + else: + feat = feat.squeeze(dim=1) + if isinstance(feat, (tuple, list)): + feat = feat[-1] + + feat = batch_gd.split_features(feat) + + if (self.seq_info_encode_layer is None and "seq_bert" not in self.word_emb_layers) or any(batch_gd.batch_graph_attributes): + if isinstance(feat, list): + feat = torch.cat(feat, -1) + + batch_gd.batch_node_features["node_feat"] = feat + + return batch_gd + else: # single-token node graph + new_feat = feat + if "seq_bert" in self.word_emb_layers: + gd_list = from_batch(batch_gd) + raw_tokens = [ + [gd.node_attributes[i]["token"] for i in range(gd.get_node_num())] + for gd in gd_list + ] + bert_feat = self.word_emb_layers["seq_bert"](raw_tokens) + bert_feat = dropout_fn( + bert_feat, self.bert_dropout, shared_axes=[-2], training=self.training + ) + new_feat.append(bert_feat) + + new_feat = torch.cat(new_feat, -1) + if self.seq_info_encode_layer is None: + batch_gd.batch_node_features["node_feat"] = new_feat + + return batch_gd + + rnn_state = self.seq_info_encode_layer( + new_feat, torch.LongTensor(batch_gd._batch_num_nodes).to(batch_gd.device) + ) + if isinstance(rnn_state, (tuple, list)): + rnn_state = rnn_state[0] + + batch_gd.batch_node_features["node_feat"] = rnn_state + + return batch_gd + +class AMRGraphEmbeddingInitialization(nn.Module): + def __init__( + self, + word_vocab, + embedding_style, + hidden_size=None, + fix_word_emb=True, + fix_bert_emb=True, + word_dropout=None, + rnn_dropout=None, + ): + super(AMRGraphEmbeddingInitialization, self).__init__() + self.embedding_layer = AmrEmbeddingConstruction( + word_vocab, + embedding_style["single_token_item"], + emb_strategy=embedding_style["emb_strategy"], + hidden_size=hidden_size, + num_rnn_layers=embedding_style.get("num_rnn_layers", 1), + fix_word_emb=fix_word_emb, + fix_bert_emb=fix_bert_emb, + bert_model_name=embedding_style.get("bert_model_name", "bert-base-uncased"), + bert_lower_case=embedding_style.get("bert_lower_case", True), + word_dropout=word_dropout, + rnn_dropout=rnn_dropout, + ) + + @abstractmethod + def forward(self, graph_data: GraphData): + return self.embedding_layer(graph_data) +class AMRDataItem(DataItem): + def __init__(self, input_text, output_text, tokenizer, output_tree=None, share_vocab=True): + super(AMRDataItem, self).__init__(input_text, tokenizer) + self.output_text = output_text + self.share_vocab = share_vocab + self.output_tree = output_tree + + def extract(self): + """ + Returns + ------- + Input tokens and output tokens + """ + g: GraphData = self.graph + + input_tokens = [] + for i in range(g.get_node_num()): + tokenized_token = self.tokenizer(g.node_attributes[i]["token"]) + input_tokens.extend(tokenized_token) + + for s in g.graph_attributes["sentence"]: + input_tokens.extend(s.strip().split(" ")) + + output_tokens = self.tokenizer(self.output_text) + + return input_tokens, output_tokens + + def extract_edge_tokens(self): + g: GraphData = self.graph + edge_tokens = [] + for i in range(g.get_edge_num()): + edge_tokens.append(g.edge_attributes[i]["token"]) + return edge_tokens + +class RGCNGraph2Tree(Graph2Tree): + def __init__( + self, + vocab_model, + embedding_style, + graph_construction_name, + # embedding + emb_input_size, + emb_hidden_size, + emb_word_dropout, + emb_rnn_dropout, + emb_fix_word_emb, + emb_fix_bert_emb, + # gnn + gnn, + gnn_num_layers, + gnn_direction_option, + gnn_input_size, + gnn_hidden_size, + gnn_output_size, + gnn_feat_drop, + gnn_attn_drop, + # decoder + dec_use_copy, + dec_hidden_size, + dec_dropout, + dec_teacher_forcing_rate, + dec_max_decoder_step, + dec_max_tree_depth, + dec_attention_type, + dec_use_sibling, + # optional + criterion=None, + share_vocab=False, + **kwargs + ): + super(RGCNGraph2Tree, self).__init__( + vocab_model=vocab_model, + embedding_style=embedding_style, + graph_construction_name=graph_construction_name, + # embedding + emb_input_size=emb_input_size, + emb_hidden_size=emb_hidden_size, + emb_word_dropout=emb_word_dropout, + emb_rnn_dropout=emb_rnn_dropout, + emb_fix_word_emb=emb_fix_word_emb, + emb_fix_bert_emb=emb_fix_bert_emb, + # gnn + gnn=gnn, + gnn_num_layers=gnn_num_layers, + gnn_direction_option=gnn_direction_option, + gnn_input_size=gnn_input_size, + gnn_hidden_size=gnn_hidden_size, + gnn_output_size=gnn_output_size, + gnn_feat_drop=gnn_feat_drop, + gnn_attn_drop=gnn_attn_drop, + # decoder + dec_use_copy=dec_use_copy, + dec_hidden_size=dec_hidden_size, + dec_dropout=dec_dropout, + dec_teacher_forcing_rate=dec_teacher_forcing_rate, + dec_max_decoder_step=dec_max_decoder_step, + dec_max_tree_depth=dec_max_tree_depth, + dec_attention_type=dec_attention_type, + dec_use_sibling=dec_use_sibling, + # optional + criterion=criterion, + share_vocab=share_vocab, + **kwargs + ) + + def _build_gnn_encoder( + self, + gnn, + num_layers, + input_size, + hidden_size, + output_size, + direction_option, + feats_dropout, + gnn_heads=None, + gnn_residual=True, + gnn_attn_dropout=0.0, + gnn_activation=F.relu, # gat + gnn_bias=True, + gnn_allow_zero_in_degree=True, + gnn_norm="both", + gnn_weight=True, + gnn_use_edge_weight=False, + gnn_gcn_norm="both", # gcn + gnn_n_etypes=1, # ggnn + gnn_aggregator_type="lstm", # graphsage + **kwargs + ): + if gnn == "rgcn": + self.gnn_encoder = RGCN( + num_layers, + input_size, + hidden_size, + output_size, + num_rels=80, + num_bases=4, + gpu=0, + ) + else: + raise NotImplementedError() + + @classmethod + def from_args(cls, opt, vocab_model): + """ + The function for building ``Graph2Tree`` model. + Parameters + ---------- + opt: dict + The configuration dict. It should has the same hierarchy and keys as the template. + vocab_model: VocabModel + The vocabulary. + + Returns + ------- + model: Graph2Tree + """ + initializer_args = cls._get_node_initializer_params(opt) + gnn_args = cls._get_gnn_params(opt) + dec_args = cls._get_decoder_params(opt) + + args = copy.deepcopy(initializer_args) + args.update(gnn_args) + args.update(dec_args) + args["share_vocab"] = opt["model_args"]["graph_construction_args"][ + "graph_construction_share" + ][ + "share_vocab" + ] # noqa + return cls(vocab_model=vocab_model, **args) + +class AMRGraph2Tree(RGCNGraph2Tree): + def __init__( + self, + vocab_model, + embedding_style, + graph_construction_name, + # embedding + emb_input_size, + emb_hidden_size, + emb_word_dropout, + emb_rnn_dropout, + emb_fix_word_emb, + emb_fix_bert_emb, + # gnn + gnn, + gnn_num_layers, + gnn_direction_option, + gnn_input_size, + gnn_hidden_size, + gnn_output_size, + gnn_feat_drop, + gnn_attn_drop, + # decoder + dec_use_copy, + dec_hidden_size, + dec_dropout, + dec_teacher_forcing_rate, + dec_max_decoder_step, + dec_max_tree_depth, + dec_attention_type, + dec_use_sibling, + # optional + criterion=None, + share_vocab=False, + **kwargs + ): + style = embedding_style["emb_strategy"] + embedding_style["emb_strategy"] = "w2v_bilstm" + super(RGCNGraph2Tree, self).__init__( + vocab_model=vocab_model, + embedding_style=embedding_style, + graph_construction_name=graph_construction_name, + # embedding + emb_input_size=emb_input_size, + emb_hidden_size=emb_hidden_size, + emb_word_dropout=emb_word_dropout, + emb_rnn_dropout=emb_rnn_dropout, + emb_fix_word_emb=emb_fix_word_emb, + emb_fix_bert_emb=emb_fix_bert_emb, + # gnn + gnn=gnn, + gnn_num_layers=gnn_num_layers, + gnn_direction_option=gnn_direction_option, + gnn_input_size=gnn_input_size, + gnn_hidden_size=gnn_hidden_size, + gnn_output_size=gnn_output_size, + gnn_feat_drop=gnn_feat_drop, + gnn_attn_drop=gnn_attn_drop, + # decoder + dec_use_copy=dec_use_copy, + dec_hidden_size=dec_hidden_size, + dec_dropout=dec_dropout, + dec_teacher_forcing_rate=dec_teacher_forcing_rate, + dec_max_decoder_step=dec_max_decoder_step, + dec_max_tree_depth=dec_max_tree_depth, + dec_attention_type=dec_attention_type, + dec_use_sibling=dec_use_sibling, + # optional + criterion=criterion, + share_vocab=share_vocab, + **kwargs + ) + embedding_style["emb_strategy"] = style + self.graph_initializer = AMRGraphEmbeddingInitialization( + word_vocab=vocab_model.in_word_vocab, + embedding_style=embedding_style, + hidden_size=emb_hidden_size, + word_dropout=emb_word_dropout, + rnn_dropout=emb_rnn_dropout, + fix_word_emb=emb_fix_word_emb, + fix_bert_emb=emb_fix_bert_emb, + ) +class InferenceText2TreeDataset(Text2TreeDataset): + def __init__( + self, + graph_construction_name: str, + root_dir: str = None, + static_or_dynamic: str = "static", + topology_builder = None, + topology_subdir: str = None, + dynamic_init_graph_name: str = None, + dynamic_init_topology_builder = None, + dynamic_init_topology_aux_args=None, + share_vocab=True, + dataitem=None, + init_edge_vocab=True, + is_hetero=True, + **kwargs, + ): + super(InferenceText2TreeDataset, self).__init__( + root_dir=root_dir, + graph_construction_name=graph_construction_name, + topology_builder=topology_builder, + topology_subdir=topology_subdir, + static_or_dynamic=static_or_dynamic, + share_vocab=share_vocab, + dynamic_init_topology_builder=dynamic_init_topology_builder, + dynamic_init_topology_aux_args=dynamic_init_topology_aux_args, + init_edge_vocab=init_edge_vocab, + is_hetero=True, + **kwargs, + ) + self.data_item_type = dataitem + + def parse_file(self, file_path) -> list: + """ + Read and parse the file specified by `file_path`. The file format is specified by + each individual task-specific base class. Returns all the indices of data items + in this file w.r.t. the whole dataset. + + For Text2TreeDataset, the format of the input file should contain lines of input, + each line representing one record of data. The input and output is separated by + a tab(\t). + + Examples + -------- + input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 ) + + DataItem: input_text="list job use languageid0", output_text="job ( ANS ) , + language ( ANS , languageid0 )" + + Parameters + ---------- + file_path: str + The path of the input file. + + Returns + ------- + list + The indices of data items in the file w.r.t. the whole dataset. + """ + data = [] + with open(file_path, "r") as f: + lines = f.readlines() + for line in lines: + input, output = line.split("\t") + data_item = self.data_item_type( + input_text=input, + output_text=output, + output_tree=None, + tokenizer=self.tokenizer, + share_vocab=self.share_vocab, + ) + data.append(data_item) + return data + + def vectorization(self, data_items): + """For tree decoder we also need the vectorize the tree output.""" + for item in data_items: + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = self.vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + + token_matrix = [] + if self.init_edge_vocab: + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = self.vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + if "pos_tag" in graph.graph_attributes: + pos_vocab = [".", "CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNP", "NNPS", "NNS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"] + pos_map = {pos: i for i, pos in enumerate(pos_vocab)} + maxlen = max(len(pos_tag) for pos_tag in graph.graph_attributes["pos_tag"]) + pos_token_id = torch.zeros(len(graph.graph_attributes["pos_tag"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["pos_tag"]): + for j, token in enumerate(sentence_token): + if token in pos_map: + pos_token_id[i][j] = pos_map[token] + else: + print('pos_tag', token) + graph.graph_attributes["pos_tag_id"] = pos_token_id + + if "entity_label" in graph.graph_attributes: + entity_label = ["O", "PERSON", "LOCATION", "ORGANIZATION", "ORGANIZATION", "MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET", "EMAIL", "URL", "CITY", "STATE_OR_PROVINCE", "COUNTRY", "NATIONALITY", "RELIGION", "TITLE", "IDEOLOGY", "CRIMINAL_CHARGE", "CAUSE_OF_DEATH", "HANDLE"] + entity_map = {entity: i for i, entity in enumerate(entity_label)} + maxlen = max(len(entity_tag) for entity_tag in graph.graph_attributes["entity_label"]) + entity_token_id = torch.zeros(len(graph.graph_attributes["entity_label"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["entity_label"]): + for j, token in enumerate(sentence_token): + if token in entity_map: + entity_token_id[i][j] = entity_map[token] + else: + print('entity_label', token) + graph.graph_attributes["entity_label_id"] = entity_token_id + + if "sentence" in graph.graph_attributes: + maxlen = max(len(sentence.strip().split()) for sentence in graph.graph_attributes["sentence"]) + seq_token_id = torch.zeros(len(graph.graph_attributes["sentence"]), maxlen, dtype=torch.long) + for i, sentence in enumerate(graph.graph_attributes["sentence"]): + sentence_token = sentence.strip().split() + for j, token in enumerate(sentence_token): + seq_token_id[i][j] = self.vocab_model.in_word_vocab.get_symbol_idx(token) + graph.graph_attributes["sentence_id"] = seq_token_id + + tgt = item.output_text + tgt_list = self.vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), self.vocab_model.out_word_vocab + ) + item.output_tree = output_tree + + def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): + item = deepcopy(data_item) + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + if hasattr(vocab_model, "edge_vocab"): + token_matrix = [] + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + + if "pos_tag" in graph.graph_attributes: + pos_vocab = [".", "CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNP", "NNPS", "NNS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"] + pos_map = {pos: i for i, pos in enumerate(pos_vocab)} + maxlen = max(len(pos_tag) for pos_tag in graph.graph_attributes["pos_tag"]) + pos_token_id = torch.zeros(len(graph.graph_attributes["pos_tag"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["pos_tag"]): + for j, token in enumerate(sentence_token): + if token in pos_map: + pos_token_id[i][j] = pos_map[token] + else: + print('pos_tag', token) + graph.graph_attributes["pos_tag_id"] = pos_token_id + + if "entity_label" in graph.graph_attributes: + entity_label = ["O", "PERSON", "LOCATION", "ORGANIZATION", "ORGANIZATION", "MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET", "EMAIL", "URL", "CITY", "STATE_OR_PROVINCE", "COUNTRY", "NATIONALITY", "RELIGION", "TITLE", "IDEOLOGY", "CRIMINAL_CHARGE", "CAUSE_OF_DEATH", "HANDLE"] + entity_map = {entity: i for i, entity in enumerate(entity_label)} + maxlen = max(len(entity_tag) for entity_tag in graph.graph_attributes["entity_label"]) + entity_token_id = torch.zeros(len(graph.graph_attributes["entity_label"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["entity_label"]): + for j, token in enumerate(sentence_token): + if token in entity_map: + entity_token_id[i][j] = entity_map[token] + else: + print('entity_label', token) + graph.graph_attributes["entity_label_id"] = entity_token_id + + if "sentence" in graph.graph_attributes: + maxlen = max(len(sentence.strip().split()) for sentence in graph.graph_attributes["sentence"]) + seq_token_id = torch.zeros(len(graph.graph_attributes["sentence"]), maxlen, dtype=torch.long) + for i, sentence in enumerate(graph.graph_attributes["sentence"]): + sentence_token = sentence.strip().split() + for j, token in enumerate(sentence_token): + seq_token_id[i][j] = vocab_model.in_word_vocab.get_symbol_idx(token) + graph.graph_attributes["sentence_id"] = seq_token_id + + if isinstance(item.output_text, str): + tgt = item.output_text + tgt_list = vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), vocab_model.out_word_vocab + ) + item.output_tree = output_tree + return item + + +class EdgeText2TreeDataset(JobsDatasetForTree): + def __init__( + self, + root_dir, + # topology_builder, + topology_subdir, + graph_construction_name, + static_or_dynamic="static", + topology_builder=None, + merge_strategy="tailhead", + edge_strategy=None, + dynamic_init_graph_name=None, + dynamic_init_topology_builder=None, + dynamic_init_topology_aux_args=None, + nlp_processor_args=None, + # pretrained_word_emb_file=None, + pretrained_word_emb_name="6B", + pretrained_word_emb_url=None, + pretrained_word_emb_cache_dir=None, + val_split_ratio=0, + word_emb_size=300, + share_vocab=True, + enc_emb_size=300, + dec_emb_size=300, + min_word_vocab_freq=1, + max_word_vocab_size=100000, + for_inference=False, + reused_vocab_model=None, + dataitem=None, + init_edge_vocab=True, + is_hetero=True, + ): + """ + Parameters + ---------- + root_dir: str + The path of dataset. + graph_name: str + The name of graph construction method. E.g., "dependency". + Note that if it is in the provided graph names (i.e., "dependency", \ + "constituency", "ie", "node_emb", "node_emb_refine"), the following \ + parameters are set by default and users can't modify them: + 1. ``topology_builder`` + 2. ``static_or_dynamic`` + If you need to customize your graph construction method, you should rename the \ + ``graph_name`` and set the parameters above. + topology_builder: GraphConstructionBase + The graph construction class. + topology_subdir: str + The directory name of processed path. + static_or_dynamic: str, default='static' + The graph type. Expected in ('static', 'dynamic') + edge_strategy: str, default=None + The edge strategy. Expected in (None, 'homogeneous', 'as_node'). + If set `None`, it will be 'homogeneous'. + merge_strategy: str, default=None + The strategy to merge sub-graphs. Expected in (None, 'tailhead', 'user_define'). + If set `None`, it will be 'tailhead'. + share_vocab: bool, default=False + Whether to share the input vocabulary with the output vocabulary. + dynamic_init_graph_name: str, default=None + The graph name of the initial graph. Expected in (None, "line", "dependency", \ + "constituency"). + Note that if it is in the provided graph names (i.e., "line", "dependency", \ + "constituency"), the following parameters are set by default and users \ + can't modify them: + 1. ``dynamic_init_topology_builder`` + If you need to customize your graph construction method, you should rename the \ + ``graph_name`` and set the parameters above. + dynamic_init_topology_builder: GraphConstructionBase + The graph construction class. + dynamic_init_topology_aux_args: None, + TBD. + """ + # Initialize the dataset. If the preprocessed files are not found, + # then do the preprocessing and save them. + super(EdgeText2TreeDataset, self).__init__( + root_dir=root_dir, + topology_builder=topology_builder, + topology_subdir=topology_subdir, + graph_construction_name=graph_construction_name, + static_or_dynamic=static_or_dynamic, + edge_strategy=edge_strategy, + merge_strategy=merge_strategy, + share_vocab=share_vocab, + pretrained_word_emb_name=pretrained_word_emb_name, + val_split_ratio=val_split_ratio, + word_emb_size=word_emb_size, + dynamic_init_graph_name=dynamic_init_graph_name, + dynamic_init_topology_builder=dynamic_init_topology_builder, + dynamic_init_topology_aux_args=dynamic_init_topology_aux_args, + nlp_processor_args=nlp_processor_args, + enc_emb_size=enc_emb_size, + dec_emb_size=dec_emb_size, + min_word_vocab_freq=min_word_vocab_freq, + max_word_vocab_size=max_word_vocab_size, + for_inference=for_inference, + reused_vocab_model=reused_vocab_model, + init_edge_vocab=init_edge_vocab, + is_hetero=is_hetero, + ) + self.data_item_type = dataitem + + @property + def processed_file_names(self): + """At least 2 reserved keys should be fiiled: 'vocab', 'data'.""" + return {"vocab": "vocab.pt", "data": "data.pt"} + + def parse_file(self, file_path) -> list: + """ + Read and parse the file specified by `file_path`. The file format is specified by + each individual task-specific base class. Returns all the indices of data items + in this file w.r.t. the whole dataset. + + For Text2TreeDataset, the format of the input file should contain lines of input, + each line representing one record of data. The input and output is separated by + a tab(\t). + + Examples + -------- + input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 ) + + DataItem: input_text="list job use languageid0", output_text="job ( ANS ) , + language ( ANS , languageid0 )" + + Parameters + ---------- + file_path: str + The path of the input file. + + Returns + ------- + list + The indices of data items in the file w.r.t. the whole dataset. + """ + data = [] + with open(file_path, "r") as f: + lines = f.readlines() + for line in lines: + input, output = line.split("\t") + data_item = self.data_item_type( + input_text=input, + output_text=output, + output_tree=None, + tokenizer=self.tokenizer, + share_vocab=self.share_vocab, + ) + data.append(data_item) + return data + + def vectorization(self, data_items): + """For tree decoder we also need the vectorize the tree output.""" + for item in data_items: + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = self.vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + + token_matrix = [] + if self.is_hetero: + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = self.vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + if "pos_tag" in graph.graph_attributes: + pos_vocab = [".", "CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNP", "NNPS", "NNS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"] + pos_map = {pos: i for i, pos in enumerate(pos_vocab)} + maxlen = max(len(pos_tag) for pos_tag in graph.graph_attributes["pos_tag"]) + pos_token_id = torch.zeros(len(graph.graph_attributes["pos_tag"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["pos_tag"]): + for j, token in enumerate(sentence_token): + if token in pos_map: + pos_token_id[i][j] = pos_map[token] + else: + print('pos_tag', token) + graph.graph_attributes["pos_tag_id"] = pos_token_id + + if "entity_label" in graph.graph_attributes: + entity_label = ["O", "PERSON", "LOCATION", "ORGANIZATION", "ORGANIZATION", "MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET", "EMAIL", "URL", "CITY", "STATE_OR_PROVINCE", "COUNTRY", "NATIONALITY", "RELIGION", "TITLE", "IDEOLOGY", "CRIMINAL_CHARGE", "CAUSE_OF_DEATH", "HANDLE"] + entity_map = {entity: i for i, entity in enumerate(entity_label)} + maxlen = max(len(entity_tag) for entity_tag in graph.graph_attributes["entity_label"]) + entity_token_id = torch.zeros(len(graph.graph_attributes["entity_label"]), maxlen, dtype=torch.long) + for i, sentence_token in enumerate(graph.graph_attributes["entity_label"]): + for j, token in enumerate(sentence_token): + if token in entity_map: + entity_token_id[i][j] = entity_map[token] + else: + print('entity_label', token) + graph.graph_attributes["entity_label_id"] = entity_token_id + + if "sentence" in graph.graph_attributes: + maxlen = max(len(sentence.strip().split()) for sentence in graph.graph_attributes["sentence"]) + seq_token_id = torch.zeros(len(graph.graph_attributes["sentence"]), maxlen, dtype=torch.long) + for i, sentence in enumerate(graph.graph_attributes["sentence"]): + sentence_token = sentence.strip().split() + for j, token in enumerate(sentence_token): + seq_token_id[i][j] = self.vocab_model.in_word_vocab.get_symbol_idx(token) + graph.graph_attributes["sentence_id"] = seq_token_id + + tgt = item.output_text + tgt_list = self.vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), self.vocab_model.out_word_vocab + ) + item.output_tree = output_tree + + def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): + item = deepcopy(data_item) + graph: GraphData = item.graph + token_matrix = [] + for node_idx in range(graph.get_node_num()): + node_token = graph.node_attributes[node_idx]["token"] + node_token_id = vocab_model.in_word_vocab.get_symbol_idx(node_token) + graph.node_attributes[node_idx]["token_id"] = node_token_id + token_matrix.append([node_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.node_features["token_id"] = token_matrix + # test if vocab_model has edge_vocab attribute + if hasattr(vocab_model, "edge_vocab"): + token_matrix = [] + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + + if isinstance(item.output_text, str): + tgt = item.output_text + tgt_list = vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) + output_tree = Tree.convert_to_tree( + tgt_list, 0, len(tgt_list), vocab_model.out_word_vocab + ) + item.output_tree = output_tree + return item \ No newline at end of file diff --git a/graph4nlp/pytorch/data/data.py b/graph4nlp/pytorch/data/data.py index c503c58f..3036eda8 100644 --- a/graph4nlp/pytorch/data/data.py +++ b/graph4nlp/pytorch/data/data.py @@ -8,7 +8,7 @@ """ import os import warnings -from collections import namedtuple +from collections import namedtuple, Counter from typing import Any, Callable, Dict, List, Tuple, Union import dgl import scipy.sparse @@ -99,6 +99,7 @@ def __init__(self, src=None, device: str = None, is_hetero: bool = False): self.batch_size = None # Batch size self._batch_num_nodes = None # Subgraph node number list with the length of batch size self._batch_num_edges = None # Subgraph edge number list with the length of batch size + self.batch_graph_attributes = [] # Subgraph attribute list with the length of batch size if src is not None: if isinstance(src, GraphData): @@ -176,7 +177,7 @@ def add_nodes(self, node_num: int, ntypes: List[str] = None): ) if not self.is_hetero: - if ntypes is None: + if ntypes is not None: raise ValueError( "The graph is homogeneous, ntypes should be None. Got {}".format(ntypes) ) @@ -878,7 +879,9 @@ def from_dgl(self, dgl_g: dgl.DGLGraph, is_hetero=False): # Add nodes self.add_nodes(dgl_g.number_of_nodes()) for k, v in dgl_g.ndata.items(): - self.node_features[k] = v + self.node_features['node_'+k] = v + + # node_features['node_embed'] -> tensor.size((num_of_node, emb_dim)) # Add edges src_tensor, tgt_tensor = dgl_g.edges() @@ -886,7 +889,9 @@ def from_dgl(self, dgl_g: dgl.DGLGraph, is_hetero=False): tgt_list = list(tgt_tensor.detach().cpu().numpy()) self.add_edges(src_list, tgt_list) for k, v in dgl_g.edata.items(): - self.edge_features[k] = v + self.edge_features['edge_'+k] = v + # edge_features['edge_emb'] -> tensor.size((number_of_edge, emb_dim)) + # edge_features['type'] -> tensor.size((number_of_edge,)) else: self.is_hetero = True # For heterogeneous DGL graphs, we perform the same routines for nodes and edges. @@ -904,19 +909,22 @@ def from_dgl(self, dgl_g: dgl.DGLGraph, is_hetero=False): # for feature_name, feature_value in node_data.items(): # self.node_features[feature_name] = feature_value node_data = dgl_g.ndata - ntypes = [] + # ntypes = [] + ntypes = [None for _ in range(dgl_g.number_of_nodes())] processed_node_types = False node_feat_dict = {} for feature_name, data_dict in node_data.items(): if not processed_node_types: for node_type, node_feature in data_dict.items(): - ntypes += [node_type] * len(node_feature) + for nidx in node_feature: + ntypes[nidx] = node_type + # ntypes += [node_type] * len(node_feature) processed_node_types = True # for node_type, node_feature in data_dict.items(): node_feat_dict[feature_name] = torch.cat(list(data_dict.values()), dim=0) self.add_nodes(len(ntypes), ntypes=ntypes) for feature_name, feature_value in node_feat_dict.items(): - self.node_features[feature_name] = feature_value + self.node_features['node_'+feature_name] = feature_value # do the same thing for edges dgl_g_etypes = dgl_g.canonical_etypes # Add edges first @@ -924,13 +932,15 @@ def from_dgl(self, dgl_g: dgl.DGLGraph, is_hetero=False): for etype in dgl_g_etypes: num_edges = dgl_g.num_edges(etype) src_type, r_type, dst_type = etype - srcs, dsts = dgl_g.find_edges( - torch.tensor(list(range(num_edges)), dtype=torch.long), etype - ) + # srcs, dsts = dgl_g.find_edges( + # torch.tensor(list(range(num_edges)), dtype=torch.long), etype + # ) + srcs, dsts = dgl_g.edges(etype=etype) srcs, dsts = ( srcs.detach().cpu().numpy().tolist(), dsts.detach().cpu().numpy().tolist(), ) + self.add_edges(srcs, dsts, etypes=[etype] * num_edges) if len(dgl_g_etypes) > 1: for feature_name, feature_dict in dgl_g.edata.items(): @@ -945,7 +955,7 @@ def from_dgl(self, dgl_g: dgl.DGLGraph, is_hetero=False): edge_feature_dict[feature_name] = feature_value # Add edge features then for feat_name, feat_value in edge_feature_dict.items(): - self.edge_features[feat_name] = feat_value + self.edge_features['edge_'+feat_name] = feat_value # edge_data = dgl_g.edata # etypes = [] # processed_edge_types = False @@ -1330,7 +1340,7 @@ def split_features(self, input_tensor: torch.Tensor, type: str = "node") -> torc return output -def from_dgl(g: dgl.DGLGraph) -> GraphData: +def from_dgl(g: dgl.DGLGraph, is_hetero=False) -> GraphData: """ Convert a dgl.DGLGraph to a GraphData object. @@ -1338,14 +1348,15 @@ def from_dgl(g: dgl.DGLGraph) -> GraphData: ---------- g : dgl.DGLGraph The source graph in DGLGraph format. - + is_hetero: bool, default=False + Whether the graph should be heterogeneous Returns ------- GraphData The converted graph in GraphData format. """ - graph = GraphData(is_hetero=not g.is_homogeneous) - graph.from_dgl(g, is_hetero=not g.is_homogeneous) + graph = GraphData(is_hetero=is_hetero) + graph.from_dgl(g, is_hetero=is_hetero) return graph @@ -1456,7 +1467,11 @@ def stack_edge_indices(gs): big_graph._batch_num_nodes = [g.get_node_num() for g in graphs] big_graph._batch_num_edges = [g.get_edge_num() for g in graphs] - # Step 8: merge node and edge types if the batch is heterograph + # Step 8: Insert graph attributes + for g in graphs: + big_graph.batch_graph_attributes.append(g.graph_attributes) + + # Step 9: merge node and edge types if the batch is heterograph if is_heterograph: node_types = [] edge_types = [] @@ -1501,6 +1516,7 @@ def from_batch(batch: GraphData) -> List[GraphData]: cum_n_edges += num_edges[i] cum_n_nodes += num_nodes[i] ret.append(g) + g.graph_attributes = batch.batch_graph_attributes[i] # Add node and edge features for k, v in batch._node_features.items(): diff --git a/graph4nlp/pytorch/data/dataset.py b/graph4nlp/pytorch/data/dataset.py index 9fe88e16..4698f00c 100644 --- a/graph4nlp/pytorch/data/dataset.py +++ b/graph4nlp/pytorch/data/dataset.py @@ -36,7 +36,7 @@ from ..modules.utils.tree_utils import Tree from ..modules.utils.tree_utils import Vocab as VocabForTree from ..modules.utils.tree_utils import VocabForAll -from ..modules.utils.vocab_utils import VocabModel +from ..modules.utils.vocab_utils import Vocab, VocabModel class DataItem(object): @@ -146,6 +146,16 @@ def extract(self): output_tokens = self.tokenizer(self.output_text) return input_tokens, output_tokens + + def extract_edge_tokens(self): + g: GraphData = self.graph + edge_tokens = [] + for i in range(g.get_edge_num()): + if "token" in g.edge_attributes[i]: + edge_tokens.append(g.edge_attributes[i]["token"]) + else: + edge_tokens.append("") + return edge_tokens class Text2LabelDataItem(DataItem): @@ -311,6 +321,8 @@ def __init__( for_inference=False, reused_vocab_model=None, nlp_processor_args=None, + init_edge_vocab=False, + is_hetero=False, **kwargs, ): """ @@ -357,6 +369,10 @@ def __init__( vocabulary data is located. nlp_processor_args: dict, default=None It contains the parameter for nlp processor such as ``stanza``. + init_edge_vocab: bool, default=False + Whether to initialize the edge vocabulary. + is_hetero: bool, default=False + Whether the graph is heterogeneous. kwargs """ super(Dataset, self).__init__() @@ -385,6 +401,8 @@ def __init__( self.topology_builder = topology_builder self.topology_subdir = topology_subdir self.use_val_for_vocab = use_val_for_vocab + self.init_edge_vocab = init_edge_vocab + self.is_hetero = is_hetero for k, v in kwargs.items(): setattr(self, k, v) self.__indices__ = None @@ -659,6 +677,7 @@ def build_vocab(self): target_pretrained_word_emb_name=self.target_pretrained_word_emb_name, target_pretrained_word_emb_url=self.target_pretrained_word_emb_url, word_emb_size=self.word_emb_size, + init_edge_vocab=self.init_edge_vocab, ) self.vocab_model = vocab_model @@ -1077,6 +1096,11 @@ def build_vocab(self): pretrained_word_emb_cache_dir=self.pretrained_word_emb_cache_dir, embedding_dims=self.dec_emb_size, ) + if self.init_edge_vocab: + all_edge_words = VocabModel.collect_edge_vocabs(data_for_vocab, self.tokenizer, lower_case=self.lower_case) + edge_vocab = Vocab(lower_case=self.lower_case, tokenizer=self.tokenizer) + edge_vocab.build_vocab(all_edge_words, max_vocab_size=None, min_vocab_freq=1) + edge_vocab.randomize_embeddings(self.word_emb_size) if self.share_vocab: all_words = Counter() @@ -1119,6 +1143,7 @@ def build_vocab(self): in_word_vocab=src_vocab_model, out_word_vocab=tgt_vocab_model, share_vocab=src_vocab_model if self.share_vocab else None, + edge_vocab=edge_vocab if self.init_edge_vocab else None, ) return self.vocab_model @@ -1136,6 +1161,18 @@ def vectorization(self, data_items): token_matrix = torch.tensor(token_matrix, dtype=torch.long) graph.node_features["token_id"] = token_matrix + if self.is_hetero: + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = self.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + tgt = item.output_text tgt_list = self.vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) output_tree = Tree.convert_to_tree( @@ -1144,7 +1181,7 @@ def vectorization(self, data_items): item.output_tree = output_tree @classmethod - def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): + def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False, is_hetero=False): item = deepcopy(data_item) graph: GraphData = item.graph token_matrix = [] @@ -1156,6 +1193,21 @@ def _vectorize_one_dataitem(cls, data_item, vocab_model, use_ie=False): token_matrix = torch.tensor(token_matrix, dtype=torch.long) graph.node_features["token_id"] = token_matrix + if is_hetero: + if not hasattr(vocab_model, "edge_vocab"): + raise ValueError("Vocab model must have edge vocab attribute") + token_matrix = [] + for edge_idx in range(graph.get_edge_num()): + if "token" in graph.edge_attributes[edge_idx]: + edge_token = graph.edge_attributes[edge_idx]["token"] + else: + edge_token = "" + edge_token_id = vocab_model.edge_vocab[edge_token] + graph.edge_attributes[edge_idx]["token_id"] = edge_token_id + token_matrix.append([edge_token_id]) + token_matrix = torch.tensor(token_matrix, dtype=torch.long) + graph.edge_features["token_id"] = token_matrix + if isinstance(item.output_text, str): tgt = item.output_text tgt_list = vocab_model.out_word_vocab.get_symbol_idx_for_list(tgt.split()) diff --git a/graph4nlp/pytorch/datasets/jobs.py b/graph4nlp/pytorch/datasets/jobs.py index 85b78eaf..5afc1c7f 100644 --- a/graph4nlp/pytorch/datasets/jobs.py +++ b/graph4nlp/pytorch/datasets/jobs.py @@ -168,6 +168,8 @@ def __init__( max_word_vocab_size=100000, for_inference=False, reused_vocab_model=None, + init_edge_vocab=False, + is_hetero=False, ): """ Parameters @@ -215,7 +217,7 @@ def __init__( # then do the preprocessing and save them. super(JobsDatasetForTree, self).__init__( root_dir=root_dir, - # topology_builder=topology_builder, + topology_builder=topology_builder, topology_subdir=topology_subdir, graph_construction_name=graph_construction_name, static_or_dynamic=static_or_dynamic, @@ -236,6 +238,8 @@ def __init__( max_word_vocab_size=max_word_vocab_size, for_inference=for_inference, reused_vocab_model=reused_vocab_model, + init_edge_vocab=init_edge_vocab, + is_hetero=is_hetero, ) diff --git a/graph4nlp/pytorch/datasets/mawps.py b/graph4nlp/pytorch/datasets/mawps.py index 5c931907..5f770749 100644 --- a/graph4nlp/pytorch/datasets/mawps.py +++ b/graph4nlp/pytorch/datasets/mawps.py @@ -50,6 +50,8 @@ def __init__( max_word_vocab_size=100000, for_inference=False, reused_vocab_model=None, + init_edge_vocab=False, + is_hetero=False, ): """ Parameters @@ -97,7 +99,7 @@ def __init__( # then do the preprocessing and save them. super(MawpsDatasetForTree, self).__init__( root_dir=root_dir, - # topology_builder=topology_builder, + topology_builder=topology_builder, topology_subdir=topology_subdir, graph_construction_name=graph_construction_name, static_or_dynamic=static_or_dynamic, @@ -118,4 +120,6 @@ def __init__( max_word_vocab_size=max_word_vocab_size, for_inference=for_inference, reused_vocab_model=reused_vocab_model, + init_edge_vocab=init_edge_vocab, + is_hetero=is_hetero, ) diff --git a/graph4nlp/pytorch/modules/utils/tree_utils.py b/graph4nlp/pytorch/modules/utils/tree_utils.py index 5e9155bb..346dca32 100644 --- a/graph4nlp/pytorch/modules/utils/tree_utils.py +++ b/graph4nlp/pytorch/modules/utils/tree_utils.py @@ -132,10 +132,11 @@ def convert_to_tree(r_list, i_left, i_right, tgt_vocab): class VocabForAll: - def __init__(self, in_word_vocab, out_word_vocab, share_vocab): + def __init__(self, in_word_vocab, out_word_vocab, share_vocab, edge_vocab=None): self.in_word_vocab = in_word_vocab self.out_word_vocab = out_word_vocab self.share_vocab = share_vocab + self.edge_vocab = edge_vocab def get_vocab_size(self): if hasattr(self, "share_vocab"): diff --git a/graph4nlp/pytorch/modules/utils/vocab_utils.py b/graph4nlp/pytorch/modules/utils/vocab_utils.py index 82c7a742..36bf0f0e 100644 --- a/graph4nlp/pytorch/modules/utils/vocab_utils.py +++ b/graph4nlp/pytorch/modules/utils/vocab_utils.py @@ -47,6 +47,8 @@ class VocabModel(object): Word embedding size, default: ``None``. share_vocab : boolean Specify whether to share vocab between input and output text, default: ``True``. + init_edge_vocab: boolean + Specify whether to initialize edge vocab, default: ``False``. Examples ------- @@ -82,6 +84,7 @@ def __init__( # pretrained_word_emb_file=None, word_emb_size=None, share_vocab=True, + init_edge_vocab=False, ): super(VocabModel, self).__init__() self.tokenizer = tokenizer @@ -150,6 +153,12 @@ def __init__( self.out_word_vocab.randomize_embeddings(word_emb_size) else: self.out_word_vocab = self.in_word_vocab + + if init_edge_vocab: + all_edge_words = VocabModel.collect_edge_vocabs(data_set, self.tokenizer, lower_case=lower_case) + self.edge_vocab = Vocab(lower_case=lower_case, tokenizer=self.tokenizer) + self.edge_vocab.build_vocab(all_edge_words, max_vocab_size=None, min_vocab_freq=1) + self.edge_vocab.randomize_embeddings(word_emb_size) if share_vocab: print("[ Initialized word embeddings: {} ]".format(self.in_word_vocab.embeddings.shape)) @@ -265,6 +274,14 @@ def collect_vocabs(all_instances, tokenizer, lower_case=True, share_vocab=True): all_words[1].update(extracted_tokens[1]) return all_words + @staticmethod + def collect_edge_vocabs(all_instances, tokenizer, lower_case=True): + """Count vocabulary tokens for edge.""" + all_edges = Counter() + for instance in all_instances: + extracted_edge_tokens = instance.extract_edge_tokens() + all_edges.update(extracted_edge_tokens) + return all_edges class WordEmbModel(Vectors): diff --git a/graph4nlp/pytorch/test/data_structure/test_graphdata.py b/graph4nlp/pytorch/test/data_structure/test_graphdata.py index 7d438715..e73ea8ad 100644 --- a/graph4nlp/pytorch/test/data_structure/test_graphdata.py +++ b/graph4nlp/pytorch/test/data_structure/test_graphdata.py @@ -324,17 +324,17 @@ def test_conversion_dgl(): def test_conversion_dgl_hetero(): g = GraphData(is_hetero=True) - g.add_nodes(10, ntypes=["A"] * 5 + ["B"] * 5) + g.add_nodes(11, ntypes=["A"] * 5 + ["B"] * 6) # g.add_nodes for i in range(5): - g.add_edge(src=i, tgt=(i + 5) % 10, etype=("A", "R_ab", "B")) + g.add_edge(src=i, tgt=(i + 5) % 11, etype=("A", "R_ab", "B")) for i in range(5): - g.add_edge(src=(i + 5) % 10, tgt=i, etype=("B", "R_ba", "A")) + g.add_edge(src=(i + 6) % 11, tgt=i, etype=("B", "R_ba", "A")) for i in range(5): g.add_edge(src=i, tgt=(i + 1) % 5, etype=("A", "R_aa", "A")) - g.node_features["node_feat"] = torch.randn((10, 10)) - g.node_features["zero"] = torch.zeros(10) - g.node_features["idx"] = torch.tensor(list(range(10)), dtype=torch.long) + g.node_features["node_feat"] = torch.randn((11, 10)) + g.node_features["zero"] = torch.zeros(11) + g.node_features["idx"] = torch.tensor(list(range(11)), dtype=torch.long) g.edge_features["edge_feat"] = torch.randn((15, 10)) g.edge_features["idx"] = torch.tensor(list(range(15)), dtype=torch.long) # Test to_dgl @@ -582,3 +582,6 @@ def test_remove_edges(): mem_report() g.remove_all_edges() mem_report() + +if __name__ == "__main__": + test_conversion_dgl_hetero() \ No newline at end of file diff --git a/graph4nlp/pytorch/test/graph_construction/test_embedding_construction.py b/graph4nlp/pytorch/test/graph_construction/test_embedding_construction.py index f0c1a6a4..d57f1d9e 100644 --- a/graph4nlp/pytorch/test/graph_construction/test_embedding_construction.py +++ b/graph4nlp/pytorch/test/graph_construction/test_embedding_construction.py @@ -1,23 +1,35 @@ import torch -from ...modules.graph_construction.embedding_construction import EmbeddingConstruction -from ...modules.utils.padding_utils import pad_2d_vals_no_size -from ...modules.utils.vocab_utils import VocabModel +from graph4nlp.pytorch.modules.graph_embedding_initialization.embedding_construction import EmbeddingConstruction +from graph4nlp.pytorch.modules.utils.padding_utils import pad_2d_vals_no_size +from graph4nlp.pytorch.modules.utils.vocab_utils import VocabModel +from graph4nlp.pytorch.data.dataset import Text2LabelDataItem +from graph4nlp.pytorch.data.data import GraphData, to_batch +from examples.pytorch.amr_graph_construction.amr_graph_construction import AMRGraphConstruction +from graph4nlp.pytorch.data.dataset import Text2LabelDataset +from graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction import DependencyBasedGraphConstruction +from stanfordcorenlp import StanfordCoreNLP if __name__ == "__main__": raw_text_data = [["I like nlp.", "Same here!"], ["I like graph.", "Same here!"]] - vocab_model = VocabModel( - raw_text_data, max_word_vocab_size=None, min_word_vocab_freq=1, word_emb_size=300 + # src_text_seq = list(zip(*raw_text_data))[0] + # src_idx_seq = [vocab_model.word_vocab.to_index_sequence(each) for each in src_text_seq] + # src_len = torch.LongTensor([len(each) for each in src_idx_seq]) + # num_seq = torch.LongTensor([len(src_len)]) + # input_tensor = torch.LongTensor(pad_2d_vals_no_size(src_idx_seq)) + # print("input_tensor: {}".format(input_tensor.shape)) + raw_data = ( + "We need to borrow 55% of the hammer price until we can get planning permission for restoration which will allow us to get a mortgage . I saw a nice dog and noticed he was eating a bone ." ) - src_text_seq = list(zip(*raw_text_data))[0] - src_idx_seq = [vocab_model.word_vocab.to_index_sequence(each) for each in src_text_seq] - src_len = torch.LongTensor([len(each) for each in src_idx_seq]) - num_seq = torch.LongTensor([len(src_len)]) - input_tensor = torch.LongTensor(pad_2d_vals_no_size(src_idx_seq)) - print("input_tensor: {}".format(input_tensor.shape)) - - emb_constructor = EmbeddingConstruction(vocab_model.word_vocab, "w2v", "bilstm", "bilstm", 128) - emb = emb_constructor(input_tensor, src_len, num_seq) - print("emb: {}".format(emb.shape)) + graph = AMRGraphConstruction.static_topology(raw_data) + data_set = Text2LabelDataItem('I like nlp.') + data_set.graph = graph + vocab_model = VocabModel( + [data_set], max_word_vocab_size=None, min_word_vocab_freq=1, word_emb_size=300 + ) + emb_constructor = EmbeddingConstruction(vocab_model.in_word_vocab, False, emb_strategy="bert_bilstm_amr",hidden_size=300) + g = Text2LabelDataset._vectorize_one_dataitem(data_set, vocab_model) + emb = emb_constructor(to_batch([g.graph, g.graph])) + print("emb: {}".format(emb.node_features)) \ No newline at end of file From 59dc4dfa1b4b1a5b9091e3d33937489ee123ce3c Mon Sep 17 00:00:00 2001 From: schenglee Date: Sun, 13 Nov 2022 01:49:53 +0800 Subject: [PATCH 2/4] update --- .../graph2tree/jobs/src_for_amr/inference.py | 17 ++++++----------- setup.py | 4 ++-- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py index 0bbfd62f..c44fd09e 100644 --- a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/inference.py @@ -11,7 +11,7 @@ import torch from utils import AMRDataItem -from graph4nlp.pytorch.datasets.mawps import MawpsDatasetForTree, tokenize_mawps +from graph4nlp.pytorch.datasets.jobs import JobsDatasetForTree, tokenize_jobs from graph4nlp.pytorch.inference_wrapper.generator_inference_wrapper_for_tree import ( GeneratorInferenceWrapper, ) @@ -21,9 +21,9 @@ warnings.filterwarnings("ignore") -class Mawps: +class Jobs: def __init__(self, opt=None): - super(Mawps, self).__init__() + super(Jobs, self).__init__() self.opt = opt seed = self.opt["env_args"]["seed"] @@ -48,7 +48,7 @@ def _build_model(self): model=self.model, beam_size=2, lower_case=True, - tokenizer=tokenize_mawps, + tokenizer=tokenize_jobs, dataset=InferenceText2TreeDataset, data_item=AMRDataItem, topology_builder=(AMRGraphConstruction if self.model.graph_construction_name == "amr" else None) @@ -57,12 +57,7 @@ def _build_model(self): @torch.no_grad() def translate(self): self.model.eval() - ret = self.inference_tool.predict( - raw_contents=[ - "2 dogs are barking . 1 more dogs start to bark . how many dogs are barking" - ], - batch_size=1, - ) + ret = self.inference_tool.predict(raw_contents=["list job on platformid0"], batch_size=1) print(ret) @@ -102,5 +97,5 @@ def print_config(config): config = load_json_config(cfg["json_config"]) # print_config(config) - runner = Mawps(opt=config) + runner = Jobs(opt=config) runner.translate() diff --git a/setup.py b/setup.py index 7124391c..65b0dcd8 100644 --- a/setup.py +++ b/setup.py @@ -65,8 +65,8 @@ def finalize_options(self): packages=find_packages( ".", exclude=( - "examples.*", - "examples", + # "examples.*", + # "examples", "graph4nlp.pytorch.test.*", "graph4nlp.pytorch.test", ), From 5361a9db2229779461f4438721b88594d3a4e1d6 Mon Sep 17 00:00:00 2001 From: schenglee Date: Sun, 13 Nov 2022 02:37:55 +0800 Subject: [PATCH 3/4] fix --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 65b0dcd8..7124391c 100644 --- a/setup.py +++ b/setup.py @@ -65,8 +65,8 @@ def finalize_options(self): packages=find_packages( ".", exclude=( - # "examples.*", - # "examples", + "examples.*", + "examples", "graph4nlp.pytorch.test.*", "graph4nlp.pytorch.test", ), From acfef4ef60566623eb15492242e3387d3e8c1a0d Mon Sep 17 00:00:00 2001 From: schenglee Date: Fri, 9 Dec 2022 16:29:39 +0800 Subject: [PATCH 4/4] fix --- .../jobs/config_for_amr/dynamic_dependency_undirected.json | 4 ++-- .../semantic_parsing/graph2tree/jobs/src_for_amr/runner.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json index 95451cef..c75d214f 100644 --- a/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/dynamic_dependency_undirected.json @@ -1,8 +1,8 @@ { "config_path": "examples/pytorch/semantic_parsing/graph2tree/jobs/config_for_amr/semantic_parsing_with_tree_decoder_dependency.yaml", "checkpoint_args.checkpoint_name": "node_emb_sage_undirected.pt", - "checkpoint_args.out_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/save_for_amr", - "env_args.gpuid": 1, + "checkpoint_args.out_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/save_for_dep", + "env_args.gpuid": 2, "training_args.batch_size": 20, "training_args.max_epochs": 150, "model_args.graph_construction_args.graph_construction_share.root_dir": "examples/pytorch/semantic_parsing/graph2tree/jobs/jobs_data", diff --git a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py index 81c5e2b4..c648da72 100644 --- a/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py +++ b/examples/pytorch/semantic_parsing/graph2tree/jobs/src_for_amr/runner.py @@ -16,7 +16,7 @@ import time import warnings from examples.pytorch.amr_graph_construction.amr_graph_construction import AMRGraphConstruction -from utils import AMRDataItem, AMRGraph2Tree, EdgeText2TreeDataset, RGCNGraph2Tree +from .utils import AMRDataItem, AMRGraph2Tree, EdgeText2TreeDataset, RGCNGraph2Tree warnings.filterwarnings("ignore")