Feature/spring boot example

crawler4j-examples/crawler4j-examples-spring-boot/README.md

@@ -0,0 +1,50 @@
+# Crawler4j Spring boot integration example
+
+On popular demand here and example of integration with Spring Boot. It has been designed to be the most idiomatic as possible
+(`@Service`, `@Async`, hibernate-validator etc).
+
+Run it:
+
+```bash
+$ mvn spring-boot:run -Dspring.profiles.active=dev
+```
+
+Submit a crawler request:
+
+```bash
+ curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{
+  "url": "http://example.com",
+  "callback": ""
+}' "http://localhost:8080/api/v1/crawl"
+```
+
+- callback: you can specify URL to callback when the crawling is done. The call will POST the response (see later) to the 
+specified callback URL.
+
+The response is similar to:
+
+```json5
+{
+    "id":1,
+    "url":"http://example.com",
+    "callback":"",
+    "started":"2018-12-15T08:09:40.436Z",
+    "status":"ACCEPTED",
+}
+```
+
+Monitor the status of a crawl request:
+
+```bash
+curl -X GET -v "http://localhost:8080/api/v1/crawl/1"
+``` 
+
+```json5
+{
+    "id":1,
+    "url":"http://example.com",
+    "callback":"",
+    "started":"2018-12-14T19:59:22.665Z",
+    "status":"DONE",
+}
+```

crawler4j-examples/crawler4j-examples-spring-boot/pom.xml

@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>crawler4j-parent</artifactId>
+        <groupId>edu.uci.ics</groupId>
+        <version>4.5.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+    <name>${project.groupId}:${project.artifactId}</name>
+    <description>Open Source Web Crawler for Java - example with Spring Boot</description>
+    <url>https://github.com/yasserg/crawler4j</url>
+    <artifactId>crawler4j-examples-spring-boot</artifactId>
+    <modelVersion>4.0.0</modelVersion>
+
+    <properties>
+        <spring.boot.version>2.1.0.RELEASE</spring.boot.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>edu.uci.ics</groupId>
+            <artifactId>crawler4j</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-devtools</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <!-- Swagger -->
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger2</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-bean-validators</artifactId>
+            <version>2.9.2</version>
+        </dependency>
+        <!-- // -->
+        <dependency>
+            <groupId>org.modelmapper</groupId>
+            <artifactId>modelmapper</artifactId>
+            <version>2.3.0</version>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>2.10.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-joda</artifactId>
+            <version>2.9.6</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.jadira.usertype/usertype.core -->
+        <dependency>
+            <groupId>org.jadira.usertype</groupId>
+            <artifactId>usertype.core</artifactId>
+            <version>6.0.1.GA</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-client</artifactId>
+            <version>9.4.12.v20180830</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+            <version>2.0.0.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>org.hibernate.validator</groupId>
+            <artifactId>hibernate-validator</artifactId>
+            <version>6.0.2.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>org.hibernate.validator</groupId>
+            <artifactId>hibernate-validator-annotation-processor</artifactId>
+            <version>6.0.2.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.4</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring.boot.version}</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <version>2.7</version>
+                <configuration>
+                    <delimiters>
+                        <delimiter>@</delimiter>
+                    </delimiters>
+                    <useDefaultDelimiters>false</useDefaultDelimiters>
+                </configuration>
+            </plugin>
+        </plugins>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+    </build>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-dependencies</artifactId>
+                <version>${spring.boot.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+</project>

crawler4j-examples/crawler4j-examples-spring-boot/src/main/java/edu/uci/ics/crawler4j/examples/spring/Application.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 Federico Tolomei <[email protected]>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.crawler4j.examples.spring;
+
+import edu.uci.ics.crawler4j.crawler.CrawlConfig;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.eclipse.jetty.client.HttpClient;
+import org.modelmapper.ModelMapper;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Scope;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.scheduling.annotation.EnableAsync;
+
+import java.nio.file.Paths;
+
+@EnableJpaRepositories
+@SpringBootApplication
+@EnableAsync
+public class Application {
+
+    public static void main(String[] args) {
+        SpringApplication.run(Application.class, args);
+    }
+
+    @Bean
+    @Scope("prototype") // Because at this point crawler4j does not guarantee CrawlConfig to be thread safe.
+    public CrawlConfig crawlerConfiguration() {
+        // See CrawlConfig for all configuration options.
+        CrawlConfig config = new CrawlConfig();
+        config.setCrawlStorageFolder(Paths.get(System.getProperty("java.io.tmpdir"), "crawler", RandomStringUtils.randomAlphabetic(5)).toString());
+        config.setIncludeBinaryContentInCrawling(false);
+        config.setProcessBinaryContentInCrawling(false);
+        config.setResumableCrawling(false);
+        config.setMaxDownloadSize(20 * 1024 * 1024);
+
+        return config;
+    }
+
+    @Bean
+    public ModelMapper modelMapper() {
+        return new ModelMapper();
+    }
+
+    @Bean(destroyMethod = "stop")
+    public HttpClient httpClient() throws Exception {
+        HttpClient client = new HttpClient();
+        client.setFollowRedirects(true);
+        client.start();
+        return client;
+    }
+
+}

crawler4j-examples/crawler4j-examples-spring-boot/src/main/java/edu/uci/ics/crawler4j/examples/spring/SpringWebCrawler.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2018 Federico Tolomei <[email protected]>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.crawler4j.examples.spring;
+
+import edu.uci.ics.crawler4j.crawler.Page;
+import edu.uci.ics.crawler4j.crawler.WebCrawler;
+import edu.uci.ics.crawler4j.examples.spring.entity.CrawlRequest;
+import edu.uci.ics.crawler4j.examples.spring.model.CrawlerRequestModel;
+import edu.uci.ics.crawler4j.examples.spring.model.CrawlerStatus;
+import edu.uci.ics.crawler4j.examples.spring.repo.CrawlerRequestRepository;
+import edu.uci.ics.crawler4j.url.WebURL;
+import org.apache.commons.lang3.StringUtils;
+import org.joda.time.DateTime;
+import org.modelmapper.ModelMapper;
+
+import java.net.URI;
+
+public class SpringWebCrawler extends WebCrawler {
+
+    private final CrawlerRequestModel request;
+    private final CrawlerRequestRepository repo;
+    private final ModelMapper modelMapper;
+
+    public SpringWebCrawler(CrawlerRequestModel request, CrawlerRequestRepository repo, ModelMapper modelMapper) {
+        this.request = request;
+        this.repo = repo;
+        this.modelMapper = modelMapper;
+    }
+
+    @Override
+    public void onStart() {
+        request.setStatus(CrawlerStatus.WORKING);
+        repo.save(modelMapper.map(request, CrawlRequest.class));
+    }
+
+    public boolean shouldVisit(Page referringPage, WebURL url) {
+        final URI uri;
+        try {
+            uri = URI.create(url.getURL());
+        } catch (IllegalArgumentException e) {
+            logger.warn("Illegal url {}", url.getURL());
+            return false;
+        }
+        if (StringUtils.startsWithIgnoreCase(referringPage.getWebURL().getURL(), uri.getScheme() + "://" + uri.getHost()) ||
+            StringUtils.startsWithIgnoreCase(referringPage.getWebURL().getURL(), "https://" + uri.getHost())) {
+            return true;
+        }
+
+        logger.debug("Ignoring " + url);
+        return false;
+    }
+    @Override
+    public void onBeforeExit() {
+        request.setStatus(CrawlerStatus.DONE);
+        request.setFinished(DateTime.now());
+        repo.save(modelMapper.map(request, CrawlRequest.class));
+    }
+
+    @Override
+    protected boolean shouldFollowLinksIn(WebURL url) {
+        return false;
+    }
+}