From ac2cf3ec27cb0c077e77debdf7900595f2b36690 Mon Sep 17 00:00:00 2001 From: bkamlapurwala Date: Mon, 10 Feb 2020 11:29:17 +0530 Subject: [PATCH 1/6] added whitlist endpoint --- .../java/focusedCrawler/rest/RestServer.java | 1 + .../rest/resources/CrawlerResource.java | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/main/java/focusedCrawler/rest/RestServer.java b/src/main/java/focusedCrawler/rest/RestServer.java index 0d6721015..fca113318 100644 --- a/src/main/java/focusedCrawler/rest/RestServer.java +++ b/src/main/java/focusedCrawler/rest/RestServer.java @@ -102,6 +102,7 @@ public void start() { server.post("/crawls/:crawler_id/labels", json(labelsResource.addLabels)); server.get( "/crawls/:crawler_id/_search", elasticsearchProxyResource.searchApi); server.post("/crawls/:crawler_id/_search", elasticsearchProxyResource.searchApi); + server.post("/crawls/:crawler_id/link_filter", json(crawlerResource.addurl)); /* * Thread management routes diff --git a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java index 6c9a847ce..2efad50eb 100644 --- a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java +++ b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java @@ -68,6 +68,46 @@ public CrawlerResource(CrawlersManager crawlManager) { return metricsManager != null ? metricsManager.getMetricsRegistry() : null; }; + public Route addurl = (request, response) -> { + try { + String crawlerId = request.params(":crawler_id"); + CrawlContext context = crawlersManager.getCrawl(crawlerId); + System.out.print(request.body()); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(request.body()); + String yaml = new YAMLMapper().writeValueAsString(jsonNode); + String path; + String directory; + if (context == null) { + response.status(HttpServletResponse.SC_NOT_FOUND); + return ImmutableMap.of("message", "Crawler not found for crawler_id " + crawlerId); + } + path = context.seedPath; + File f = new File(path); + directory = f.getParent(); + String link_filters_path = directory + File.separator + "link_filters.yaml"; + System.out.println(link_filters_path); + File link_filter_file = new File(link_filters_path); + link_filter_file.delete(); + //create new file with content + link_filter_file.createNewFile(); + + FileWriter myWriter = new FileWriter(link_filter_file); + //FileWriter myWriter = new FileWriter(link_filters_path); + myWriter.write(yaml); + myWriter.close(); + System.out.println("Successfully wrote to the file."); + return ImmutableMap.of("output", yaml); + + } catch (Exception e) { + logger.error("failed.", e); + response.status(HttpServletResponse.SC_NOT_ACCEPTABLE); + System.out.println("hello"); + return ImmutableMap.of("message", false); + + } + }; + public Route startCrawl = (request, response) -> { try { String crawlerId = request.params(":crawler_id"); From 563031b5044bcb523f83d3da1703ff06ed8202a4 Mon Sep 17 00:00:00 2001 From: bkamlapurwala Date: Mon, 10 Feb 2020 12:46:20 +0530 Subject: [PATCH 2/6] added import library --- .../rest/resources/CrawlerResource.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java index 2efad50eb..1ee898787 100644 --- a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java +++ b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java @@ -6,6 +6,7 @@ import java.util.Optional; import javax.servlet.http.HttpServletResponse; +import com.fasterxml.jackson.core.JsonParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +14,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import focusedCrawler.Main; import focusedCrawler.crawler.CrawlersManager; @@ -23,6 +26,17 @@ import focusedCrawler.util.MetricsManager; import spark.Request; import spark.Route; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.concurrent.Immutable; public class CrawlerResource { From 095eb5ef0717a1b809a6da9aac2b636316bb37bb Mon Sep 17 00:00:00 2001 From: root Date: Tue, 11 Feb 2020 19:26:43 +0000 Subject: [PATCH 3/6] Added http(s) protocol to Elasticsearch Rest endpoint to avoid (connection refused) error --- .../target/repository/ElasticSearchRestTargetRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/focusedCrawler/target/repository/ElasticSearchRestTargetRepository.java b/src/main/java/focusedCrawler/target/repository/ElasticSearchRestTargetRepository.java index 91b6148f6..cc2b399fa 100644 --- a/src/main/java/focusedCrawler/target/repository/ElasticSearchRestTargetRepository.java +++ b/src/main/java/focusedCrawler/target/repository/ElasticSearchRestTargetRepository.java @@ -193,7 +193,7 @@ public RestClient createRestClient(ElasticSearchConfig config) { for (String host : esHosts) { try { URL url = new URL(host); - hosts.add(new HttpHost(url.getHost(), url.getPort())); + hosts.add(new HttpHost(url.getHost(), url.getPort(), url.getProtocol())); } catch (MalformedURLException e) { throw new RuntimeException("Failed to initialize Elasticsearch REST client. " + "Invalid host: " + host, e); From e4e7798b4261fb6693977ec18daa7446f52c7aa1 Mon Sep 17 00:00:00 2001 From: Him754 Date: Mon, 30 Mar 2020 16:53:23 +0530 Subject: [PATCH 4/6] restart --- .../java/focusedCrawler/rest/RestServer.java | 3 +- .../rest/resources/CrawlerResource.java | 54 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/main/java/focusedCrawler/rest/RestServer.java b/src/main/java/focusedCrawler/rest/RestServer.java index fca113318..4bc24e8a2 100644 --- a/src/main/java/focusedCrawler/rest/RestServer.java +++ b/src/main/java/focusedCrawler/rest/RestServer.java @@ -103,7 +103,8 @@ public void start() { server.get( "/crawls/:crawler_id/_search", elasticsearchProxyResource.searchApi); server.post("/crawls/:crawler_id/_search", elasticsearchProxyResource.searchApi); server.post("/crawls/:crawler_id/link_filter", json(crawlerResource.addurl)); - + server.post("/crawls/:crawler_id/restartCrawl", json(crawlerResource.restartCrawl)); + server.get("/crawl/helloWorld", json(crawlerResource.helloWorld)); /* * Thread management routes */ diff --git a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java index 1ee898787..e4edfc9e8 100644 --- a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java +++ b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java @@ -37,8 +37,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.concurrent.Immutable; +import org.apache.commons.io.FileUtils; +import java.util.concurrent.TimeUnit ; public class CrawlerResource { + private Map crawlers = new HashMap<>(); public static final String VERSION = Main.class.getPackage().getImplementationVersion(); @@ -68,6 +71,10 @@ public CrawlerResource(CrawlersManager crawlManager) { Map crawlers = crawlersManager.getCrawls(); return ImmutableMap.of("crawlers", crawlers.values()); }; + public Route helloWorld = (request, response) -> { + System.out.println("helloWorld"); + return ImmutableMap.of("crawlers", "hello this is working"); + }; public Route metricsResource = (request, response) -> { String crawlerId = request.params(":crawler_id"); @@ -85,7 +92,8 @@ public CrawlerResource(CrawlersManager crawlManager) { public Route addurl = (request, response) -> { try { String crawlerId = request.params(":crawler_id"); - CrawlContext context = crawlersManager.getCrawl(crawlerId); + CrawlContext context = crawlersManager.getCrawl(crawlerId); + System.out.println(context); System.out.print(request.body()); ObjectMapper mapper = new ObjectMapper(); JsonNode jsonNode = mapper.readTree(request.body()); @@ -125,7 +133,6 @@ public CrawlerResource(CrawlersManager crawlManager) { public Route startCrawl = (request, response) -> { try { String crawlerId = request.params(":crawler_id"); - StartCrawlParams params = json.readValue(request.body(), StartCrawlParams.class); crawlersManager.createCrawler(crawlerId, params); @@ -143,6 +150,49 @@ public CrawlerResource(CrawlersManager crawlManager) { "crawlerStarted", false); } }; + public Route restartCrawl = (request, response) -> { + try { + String crawlerId = request.params(":crawler_id"); + StartCrawlParams params = json.readValue(request.body(), StartCrawlParams.class); + CrawlContext context = crawlersManager.getCrawl(crawlerId); + System.out.println(context); + String path; + String directory; + if (context == null) { + response.status(HttpServletResponse.SC_NOT_FOUND); + return ImmutableMap.of("message", "Crawler not found for crawler_id " + crawlerId); + } + + path = context.dataPath; + File f = new File(path); + directory = f.getParent(); + String crawler_path = directory + File.separator + crawlerId; + File crawler_directory = new File(crawler_path); + + AsyncCrawler crawler = context.getCrawler(); + crawler.stopAsync(); + crawler.awaitTerminated(); + if (crawler_directory.exists()) { + FileUtils.cleanDirectory(crawler_directory); + FileUtils.forceDelete(crawler_directory); + System.out.println("folder deleted"); + } + crawlersManager.createCrawler(crawlerId, params); + crawlersManager.startCrawl(crawlerId); + System.out.println("Crawler stopped successfully"); + + return ImmutableMap.of( + "message", "Crawler restarted successfully.", + "crawlerreStarted", true); + + } catch (Exception e) { + logger.error("Failed to restart crawler.", e); + response.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return ImmutableMap.of( + "message", "Failed to restart crawler.", + "crawlerreStarted", false); + } + }; public Route stopCrawl = (request, response) -> { try { From 20936bbb9f9057cdb4e543ea8edb23322f96c54d Mon Sep 17 00:00:00 2001 From: Him754 Date: Mon, 30 Mar 2020 17:23:27 +0530 Subject: [PATCH 5/6] added_restart_api --- src/main/java/focusedCrawler/rest/RestServer.java | 3 +-- .../java/focusedCrawler/rest/resources/CrawlerResource.java | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/java/focusedCrawler/rest/RestServer.java b/src/main/java/focusedCrawler/rest/RestServer.java index 4bc24e8a2..065ff90cf 100644 --- a/src/main/java/focusedCrawler/rest/RestServer.java +++ b/src/main/java/focusedCrawler/rest/RestServer.java @@ -104,7 +104,6 @@ public void start() { server.post("/crawls/:crawler_id/_search", elasticsearchProxyResource.searchApi); server.post("/crawls/:crawler_id/link_filter", json(crawlerResource.addurl)); server.post("/crawls/:crawler_id/restartCrawl", json(crawlerResource.restartCrawl)); - server.get("/crawl/helloWorld", json(crawlerResource.helloWorld)); /* * Thread management routes */ @@ -163,7 +162,7 @@ private void enableCORS(final String origin, final String methods) { return "OK"; }); - server.before((request, response) -> { + server.before((request, returnesponse) -> { response.header("Access-Control-Allow-Origin", origin); response.header("Access-Control-Request-Method", methods); }); diff --git a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java index e4edfc9e8..4ea7e3116 100644 --- a/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java +++ b/src/main/java/focusedCrawler/rest/resources/CrawlerResource.java @@ -71,11 +71,7 @@ public CrawlerResource(CrawlersManager crawlManager) { Map crawlers = crawlersManager.getCrawls(); return ImmutableMap.of("crawlers", crawlers.values()); }; - public Route helloWorld = (request, response) -> { - System.out.println("helloWorld"); - return ImmutableMap.of("crawlers", "hello this is working"); - }; - + public Route metricsResource = (request, response) -> { String crawlerId = request.params(":crawler_id"); From 52abf9443588ab345f3aebf44fb9f6fda62632aa Mon Sep 17 00:00:00 2001 From: Him754 Date: Mon, 30 Mar 2020 19:50:32 +0530 Subject: [PATCH 6/6] restart_api --- src/main/java/focusedCrawler/rest/RestServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/focusedCrawler/rest/RestServer.java b/src/main/java/focusedCrawler/rest/RestServer.java index 065ff90cf..075d8e40a 100644 --- a/src/main/java/focusedCrawler/rest/RestServer.java +++ b/src/main/java/focusedCrawler/rest/RestServer.java @@ -162,7 +162,7 @@ private void enableCORS(final String origin, final String methods) { return "OK"; }); - server.before((request, returnesponse) -> { + server.before((request, response) -> { response.header("Access-Control-Allow-Origin", origin); response.header("Access-Control-Request-Method", methods); });