diff --git a/README.md b/README.md index 37da1a2..114786c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The Edison Web interface is a Node-based web application that allows one to sear This application requires the following to generate its components and to run: - node - - npm (node package manager) + - npm (node package manager - version >= 3.8.3) - bash - java (version > 1.6) - mvn @@ -13,13 +13,15 @@ This application requires the following to generate its components and to run: Steps to run the application: 1) Extract the javadoc from the online javadoc repository. - - Replace the base_javadoc_url variable with the appropriate base_url (The url that produces the API overview page) + - Replace the base_javadoc_url variable in the 'extract_javadoc.sh' script in the javadoc_scraper folder with the appropriate base_url (The url that produces the API overview page) - The current default is: "http://cogcomp.cs.illinois.edu/software/doc/illinois-edison/apidocs/" - Run './extract_javadoc.sh' from the 'javadoc_scraper' directory 2) Use the jdparser maven package to parser and generate the 'data.json' file that will be ingested and presented by the web interface. - Ensure that the base_url variable in the 'src/test/java/cz/zcu/validator/jdparser/JDParserTest.java' file contains the base url with the root package path of the LREC feature classes. - The current default is: "http://cogcomp.cs.illinois.edu/software/doc/apidocs/edu/illinois/cs/cogcomp/edison/features/" + - Ensure that the github_url_base_raw variable in the 'src/test/java/cz/zcu/validator/jdparser/JDParserTest.java' file contains the base raw github url from which we would append a "Test[FeatureExtractorName]" to download the Test file to be presented. + - The current default is: "https://raw.githubusercontent.com/IllinoisCogComp/illinois-cogcomp-nlp/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/" - Run 'mvn test' from the 'javadoc_scraper' directory, and this will generate the 'data.json' file in the base directory of our web application 3) Start the Web Application @@ -32,4 +34,8 @@ Steps to run the application: - jade - bootstrap - [jquery-searchable](https://github.com/stidges/jquery-searchable) - \ No newline at end of file + +##Troubleshooting + - 'Error: Cannot find module 'npmlog'. + - This error usually occurs in outdated versions of npm (typically < 2.5.8) so be sure to run 'sudo npm install npm -g' to fix this issue. + - This error could also occur in some Mac installations and the best means to resolve the issue in that case is to a well-supported package installer such as 'homebrew' to handle your npm installation on Mac \ No newline at end of file diff --git a/javadoc_scraper/robots.txt b/javadoc_scraper/robots.txt new file mode 100644 index 0000000..7d329b1 --- /dev/null +++ b/javadoc_scraper/robots.txt @@ -0,0 +1 @@ +User-agent: * diff --git a/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/JsoupJavaDocParser.java b/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/JsoupJavaDocParser.java index 743c0f4..b9d461c 100755 --- a/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/JsoupJavaDocParser.java +++ b/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/JsoupJavaDocParser.java @@ -109,17 +109,18 @@ public void setJavadoc_url(String javadoc_url) { */ private JavaDocsFactory javaDocFactory = null; - private String base_url, repo_path_string, path_to_write; + private String base_url, repo_path_string, base_github_url_raw, path_to_write; //Old approach to extracting classes that we require. //String[] fexes = {"LabelOneAfter", "LabelOneBefore", "LabelTwoAfter", "POSWindow"}; - public JsoupJavaDocParser(String base_url, String repo_path_string, String path_to_write) { + public JsoupJavaDocParser(String base_url, String repo_path_string, String base_github_url_raw, String path_to_write) { this.skeletonFactory = new SkeletonFactory(); this.javaDocFactory = new JavaDocsFactory(); this.base_url = base_url; this.repo_path_string = repo_path_string; + this.base_github_url_raw = base_github_url_raw; this.path_to_write = path_to_write; } @@ -202,13 +203,13 @@ public Collection parseDataFromJavaDoc( ClassRepresentation a = (ClassRepresentation) reps[i]; String java_code = ""; - String base_raw_url = "https://raw.githubusercontent.com/pauljv92/testedison/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/"; + //String base_raw_url = "https://raw.githubusercontent.com/pauljv92/testedison/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/"; JavaDocClass jdoc = a.getJavaDocClass(); try { - URL url = new URL(base_raw_url+"Test"+jdoc.getName()+".java"); + URL url = new URL(this.base_github_url_raw+"Test"+jdoc.getName()+".java"); Scanner s = new Scanner(url.openStream()); while(s.hasNext()) { // operate diff --git a/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/Main.java b/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/Main.java index 76f7fa0..a3da04c 100755 --- a/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/Main.java +++ b/javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/Main.java @@ -17,7 +17,7 @@ public class Main { */ public static void main(String[] args) { - JsoupJavaDocParser parser = new JsoupJavaDocParser("None", "/test/","../data.json"); + JsoupJavaDocParser parser = new JsoupJavaDocParser("None", "/test/","../data.json", "http://github.com/test"); try { diff --git a/javadoc_scraper/src/test/java/cz/zcu/validator/jdparser/JDParserTest.java b/javadoc_scraper/src/test/java/cz/zcu/validator/jdparser/JDParserTest.java index 9de8808..84cc7d0 100755 --- a/javadoc_scraper/src/test/java/cz/zcu/validator/jdparser/JDParserTest.java +++ b/javadoc_scraper/src/test/java/cz/zcu/validator/jdparser/JDParserTest.java @@ -20,10 +20,13 @@ public void standardJavaDocDocumentsTest() { String base_url = "http://cogcomp.cs.illinois.edu/software/doc/apidocs/edu/illinois/cs/cogcomp/edison/features/"; //This URL will be expected to the base URL of the public repository. We will extract the testcode to be presented on the web interface from here. String repository_path = "/lrec/"; //We will be using string to check for Classes whose path contains this substring. - + + //Raw Github URL base for Tests + String github_url_base_raw = "https://raw.githubusercontent.com/IllinoisCogComp/illinois-cogcomp-nlp/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/"; + String path_to_write_json = "../data.json"; - JsoupJavaDocParser parser = new JsoupJavaDocParser(base_url, repository_path, path_to_write_json); + JsoupJavaDocParser parser = new JsoupJavaDocParser(base_url, repository_path, github_url_base_raw, path_to_write_json); try { // parses javadoc directory of the Edison project