log inhelp

View
 

API Sample Code

Page history last edited by Aliya 1 month, 1 week ago

Table of Contents

  1. Java API Sample Code
    1. Set Up API Profile
    2. Create a Job
    3. Upload Seedlist and Create a Job
    4. Generate a JobSetting Object
      1. Basic Job Object with REGULAR_EXPRESSION_LIST as Analysis Method
      2. Job Object with EIGHTY_APP as Analysis Method
    5. Download Job Results  
    6.  Using Other API Operations
    7. Delete Job After Downloading Results   
    8. Download new results for a Crawl Package
    9. Download results for Crawl Package - Shows how to get Jobs and JobRuns

 

 

Java API Sample Code

Set Up API Profile

The following source code example illustrates how to use the API profile and instantiate the API's JavaEightyLegsConnector object:

 

     // Instantiate an instance of APIProfile and set the following parameters:

     // 1. VersionId - Set to "1.0".

     // 2. ApiToken - Set this to the value you obtained from the web portal after following the instructions in the Getting Started section.

APIProfile profile = new APIProfile();

profile.setVersion("1.0");

profile.setApiToken("[Your API Token]"); 

 

// Instantiate an instance of JavaEightyLegsConnector using the APIProfile as the parameter. 

IEightyLegsConnector connector = new JavaEightyLegsConnector(profile);

 

 

Create a Job

The following source code example illustrates how to upload a jar and use it to create a simple job through the 80legs API:

 

package test.api;

 

import java.io.File;

import java.io.IOException; 

 

import com.eightylegs.common.domain.APIProfile;

import com.eightylegs.common.domain.AnalysisMethod;

import com.eightylegs.common.domain.AnalysisSetting;

import com.eightylegs.common.domain.CrawlSetting;

import com.eightylegs.common.domain.EnvironmentType;

import com.eightylegs.common.domain.FrequencyType;

import com.eightylegs.common.domain.JobSetting;

import com.eightylegs.common.domain.ResultSetting;

import com.eightylegs.common.domain.ResultType;

import com.eightylegs.common.domain.WebServiceEnvironment;

import com.eightylegs.common.exception.EightyLegsParameterException;

import com.eightylegs.common.exception.EightyLegsXmlException;

import com.eightylegs.customer.EightyLegsRuntime;

import com.eightylegs.customer.api.IEightyLegsConnector;

import com.eightylegs.customer.api.JavaEightyLegsConnector;

import com.eightylegs.customer.exception.EightyLegsAPIException;

import com.eightylegs.customer.exception.EightyLegsConnectionException;

import com.eightylegs.customer.exception.EightyLegsException;

import com.eightylegs.customer.exception.EightyLegsFileNotFoundException;

import com.eightylegs.customer.exception.EightyLegsIOException; 

 

public class TestUploadCodeCreateAJobAPI { 

 

      public static void main(String[] args) throws IOException { 

 

            try {

                  APIProfile profile = new APIProfile();

                  profile.setVersion("1.0");

                  profile.setApiToken("[Your API Token]");

 

                  IEightyLegsConnector connector = new JavaEightyLegsConnector(profile);

 

                  String codeFileName = "80legsWebAnalysisCode.jar";

                  int uploadedCodeId = -1; 

 

                  File f = new File("C://test//" + codeFileName);

                  uploadedCodeId = connector.uploadCode(f, "Sample Code", 50); //Note : 3rd parameter specifies the maximum number of heap space required for the code to run

  

                  JobSetting job = new JobSetting();

                  job.setName("Sample Job");

                  job.setEnvironmentType(EnvironmentType.LIVE);

                  job.setFrequencyType(FrequencyType.DOES_NOT_REPEAT); 

 

                                   // Optional.  Provides recurring options.  If the frequency type is set to DOES_NOT_REPEAT, the interval is defaulted to -1. 

                  // Otherwise, the interval is defaulted to 1. In the other cases, the interval should be greater than or equal to 1.

                                    job.setFrequencyInterval(-1);

 

                  // Crawl Settings

                  CrawlSetting crawlRequest = new CrawlSetting();

 

                  // Use only one starting crawl url

                  crawlRequest.getSeedList().add("http://dir.yahoo.com");

                  crawlRequest.setCrawlRegularExpression("^.+crawlexpression.com.*");

                                    crawlRequest.setCrawlType(CrawlType.FAST);

             crawlRequest.getMimeTypeList().add("text");

             crawlRequest.setOutgoingLinkToCrawl(OutgoingLinkType.CRAWL_ALL_LINKS);

             crawlRequest.setPreservingQueryStringWhenCrawling(true);

             crawlRequest.setMaxNumberOfUrls(100);

             crawlRequest.setMaxNumberUrlsPerPage(100);

             crawlRequest.setDepthLevel(1);

             job.setCrawlSetting(crawlRequest);

 

                  // Analysis Settings

                  AnalysisSetting analysisReq = new AnalysisSetting();

                  analysisReq.setAnalysisRegularExpression("^.+yahoo\\.com.*");

                  analysisReq.getMimeTypeList().add("text");

                  analysisReq.setAnalysisMethod(AnalysisMethod.CODE);

 

                  //Note: analysisMethodList should only be set if AnalysisMethod is REGULAR_EXPRESSION_LIST or KEYWORD_LIST

                  //analysisReq.getAnalysisMethodList().add("yahoo");

 

                  analysisReq.setCodeId(uploadedCodeId);  

                  // Note: dataId is not required.

                  job.setAnalysisSetting(analysisReq); 

 

                  // Result Settings

                  ResultSetting resultSetting = new ResultSetting();

                                    // Build-in Keyword/Pattern Matching usess UNIQUE_AND_TOTAL_COUNT ResultType, optional of BOOLEAN_ARRAY and COUNT_ARRAY are available

                  resultSetting.setResultType(ResultType.CODE_RESULTS);

                  resultSetting.setHasCrawlUrlsInResult(true);

                  resultSetting.setMaxResultFileSizeInMB(100);

                  job.setResultSetting(resultSetting); 

 

                  // Calls the API create job method to create the job

                  int jobId = connector.createJob(job);

                  System.out.println("Created job id: " + jobId);  

            } catch (EightyLegsAPIException e) {

                  e.printStackTrace();

            } catch (EightyLegsParameterException e) {

                  e.printStackTrace();

            } catch (EightyLegsXmlException e) {

                  e.printStackTrace();

            } catch (EightyLegsIOException e) {

                  e.printStackTrace();

            } catch (EightyLegsException e) {

                  e.printStackTrace();

            } catch (EightyLegsConnectionException e) {

                  e.printStackTrace();

            } catch (EightyLegsFileNotFoundException e) {

                  e.printStackTrace();

            }

      }

} 

 

Upload Seedlist and Create a Job

The following source code example illustrates how to upload a jar and use it to create a simple job through the 80legs API:

 

package test.api;

 

import java.io.File;

import java.io.IOException; 

 

import com.eightylegs.common.domain.APIProfile;

import com.eightylegs.common.domain.AnalysisMethod;

import com.eightylegs.common.domain.AnalysisSetting;

import com.eightylegs.common.domain.CrawlSetting;

import com.eightylegs.common.domain.EnvironmentType;

import com.eightylegs.common.domain.FrequencyType;

import com.eightylegs.common.domain.JobSetting;

import com.eightylegs.common.domain.ResultSetting;

import com.eightylegs.common.domain.ResultType;

import com.eightylegs.common.domain.WebServiceEnvironment;

import com.eightylegs.common.exception.EightyLegsParameterException;

import com.eightylegs.common.exception.EightyLegsXmlException;

import com.eightylegs.customer.EightyLegsRuntime;

import com.eightylegs.customer.api.IEightyLegsConnector;

import com.eightylegs.customer.api.JavaEightyLegsConnector;

import com.eightylegs.customer.exception.EightyLegsAPIException;

import com.eightylegs.customer.exception.EightyLegsConnectionException;

import com.eightylegs.customer.exception.EightyLegsException;

import com.eightylegs.customer.exception.EightyLegsFileNotFoundException;

import com.eightylegs.customer.exception.EightyLegsIOException; 

 

public class TestUploadCodeCreateAJobAPI { 

 

      public static void main(String[] args) throws IOException { 

 

            try {

                  APIProfile profile = new APIProfile();

                  profile.setVersion("1.0");

                  profile.setApiToken("[Your API Token]");

 

                  IEightyLegsConnector connector = new JavaEightyLegsConnector(profile);

 

                  String seedlistFileName = "seeds.txt";

                  int uploadedSeedlistId = -1; 

 

                  File f = new File("C://test//" + seedlistFileName );

                  uploadedSeedlistId = connector.uploadSeedList(f, f.getName() + Calendar.getInstance().getTimeInMillis(), true); //Note : 3rd parameter specifies whether bad urls should be ignored.

 

                  JobSetting job = new JobSetting();

                  job.setName("Sample Job");

                  job.setEnvironmentType(EnvironmentType.LIVE);

                  job.setFrequencyType(FrequencyType.DOES_NOT_REPEAT); 

 

                                   // Optional.  Provides recurring options.  If the frequency type is set to DOES_NOT_REPEAT, the interval is defaulted to -1. 

                  // Otherwise, the interval is defaulted to 1. In the other cases, the interval should be greater than or equal to 1.

                                    job.setFrequencyInterval(-1);

 

                  // Crawl Settings

                  CrawlSetting crawlRequest = new CrawlSetting();

 

                  // Use only one starting crawl url

                  crawlRequest.getSeedListId(uploadedSeedlistId);

                  crawlRequest.setCrawlRegularExpression("^.+crawlexpression.com.*");

                                    crawlRequest.setCrawlType(CrawlType.FAST);

                       crawlRequest.getMimeTypeList().add("text");

                       crawlRequest.setOutgoingLinkToCrawl(OutgoingLinkType.CRAWL_ALL_LINKS);

                       crawlRequest.setPreservingQueryStringWhenCrawling(true);

                       crawlRequest.setMaxNumberOfUrls(100);

                       crawlRequest.setMaxNumberUrlsPerPage(100);

                       crawlRequest.setDepthLevel(1);

                       job.setCrawlSetting(crawlRequest);

 

                  // Analysis Settings

                  AnalysisSetting analysisReq = new AnalysisSetting();

                  analysisReq.setAnalysisRegularExpression("");

                  analysisReq.getMimeTypeList().add("text");

                  analysisReq.setAnalysisMethod(AnalysisMethod.KEYWORD_LIST);

 

                  //Note: analysisMethodList should only be set if AnalysisMethod is REGULAR_EXPRESSION_LIST or KEYWORD_LIST

                  analysisReq.getAnalysisMethodList().add("80legs");

 

                  job.setAnalysisSetting(analysisReq); 

 

                  // Result Settings

                  ResultSetting resultSetting = new ResultSetting();

                                    // Build-in Keyword/Pattern Matching usess UNIQUE_AND_TOTAL_COUNT ResultType, optional of BOOLEAN_ARRAY and COUNT_ARRAY are available

                  resultSetting.setResultType(ResultType.COUNT_ARRAY);

                  resultSetting.setHasCrawlUrlsInResult(true);

                  resultSetting.setMaxResultFileSizeInMB(100);

                  job.setResultSetting(resultSetting); 

 

                  // Calls the API create job method to create the job

                  int jobId = connector.createJob(job);

                  System.out.println("Created job id: " + jobId); 

            } catch (EightyLegsAPIException e) {

                  e.printStackTrace();

            } catch (EightyLegsParameterException e) {

                  e.printStackTrace();

            } catch (EightyLegsXmlException e) {

                  e.printStackTrace();

            } catch (EightyLegsIOException e) {

                  e.printStackTrace();

            } catch (EightyLegsException e) {

                  e.printStackTrace();

            } catch (EightyLegsConnectionException e) {

                  e.printStackTrace();

            } catch (EightyLegsFileNotFoundException e) {

                  e.printStackTrace();

            }

      }

}

Generate a JobSetting Object

 

Basic Job Object with REGULAR_EXPRESSION_LIST as Analysis Method

 

The JobSetting object is a basic job object that has the fields required to create a job.

 

public static JobSetting generateJob() {

 

JobSetting job = new JobSetting();

job.setName("Sample Job");

job.setEnvironmentType(EnvironmentType.LIVE);

job.setFrequencyType(FrequencyType.DOES_NOT_REPEAT);

 

          // Optional.  Provides recurring options.  If the frequency type is set to DOES_NOT_REPEAT, the interval is defaulted to -1. 

          // Otherwise, the interval is defaulted to 1. In the other cases, the interval should be greater than or equal to 1.

          job.setFrequencyInterval(-1);

 

// Crawl Settings

CrawlSetting crawlRequest = new CrawlSetting();

 

// There are two ways to specify seed lists:

// (1) Add URLs to seedList OR

// (2) Specify an already uploaded seedlist

// The following is example of (1):

crawlRequest.getSeedList().add("http://dir.yahoo.com");

 

// The following is an example of (2):

// crawlRequest.setSeedListId(seedListId); 

 

crawlRequest.setCrawlRegularExpression("^.+crawlexpression.com.*");

crawlRequest.setCrawlType(CrawlType.FAST);

crawlRequest.getMimeTypeList().add("text");

crawlRequest.setOutgoingLinkToCrawl(OutgoingLinkType.CRAWL_ALL_LINKS);

crawlRequest.setPreservingQueryStringWhenCrawling(true);

crawlRequest.setMaxNumberOfUrls(100);

crawlRequest.setMaxNumberUrlsPerPage(100);

crawlRequest.setDepthLevel(1);

job.setCrawlSetting(crawlRequest);

 

// Analysis Settings

AnalysisSetting analysisReq = new AnalysisSetting();

analysisReq.setAnalysisRegularExpression("^.+yahoo\\.com.*");

analysisReq.getMimeTypeList().add("text");

analysisReq.setAnalysisMethod(AnalysisMethod.REGULAR_EXPRESSION_LIST);

analysisReq.getAnalysisMethodList().add("yahoo");

job.setAnalysisSetting(analysisReq);

 

// Result Settings

ResultSetting resultSetting = new ResultSetting();

//Built-in Keyword/Pattern Matching uses UNIQUE_AND_TOTAL_COUNT ResultType, optional of BOOLEAN_ARRAY and COUNT_ARRAY are available

resultSetting.setResultType(ResultType.COUNT_ARRAY);

resultSetting.setHasCrawlUrlsInResult(true);

resultSetting.setMaxResultFileSizeInMB(100);

job.setResultSetting(resultSetting);

 

return job;

} 

 

Job Object with EIGHTY_APP as Analysis Method

 

First, you will need an 80app Version Id for the Eighty App that are you may want to use.  We are working on a page for the Portal that will tell you all of your 80app version ids, but for now, you can find your available ids by doing this: 

  • Login to the Portal

  • Click on the "Marketplace" tab.

  • Select "Show Mine" link under 80apps section.

  • Find the 80app under "80app Version Details" column.  The latest Eighty App Version Id is next to the 80app.  Note:  Please do not use setEightyAppId and instead use setEightyAppVersionId.  If you want to use an 80app that doesn't show up in this list, click on the "Marketplace" tab and purchase the 80app that you want to use (many of them are free, but you have to "purchase" them anyway).

  • If the 80app has the following usage configuration: "This 80app requires multiple lines of configuration data."  Upload the configuration using the data file.  This file can be uploaded just before the job is created and assigned to the job or it could have already been created and used later. 
     

public static JobSetting generateEightyAppJob() {

 

JobSetting job = new JobSetting();

job.setName("Sample Job using Eighty App");

job.setEnvironmentType(EnvironmentType.LIVE);

job.setFrequencyType(FrequencyType.DOES_NOT_REPEAT);

 

          // Optional.  Provides recurring options.  If the frequency type is set to DOES_NOT_REPEAT, the interval is defaulted to -1. 

          // Otherwise, the interval is defaulted to 1. In the other cases, the interval should be greater than or equal to 1.

          job.setFrequencyInterval(-1);

 

// Crawl Settings

CrawlSetting crawlRequest = new CrawlSetting();

 

// There are two ways to specify seed lists:

// (1) Add URLs to seedList OR

// (2) Specify an already uploaded seedlist

// The following is example of (1):

crawlRequest.getSeedList().add("http://dir.yahoo.com");

 

// The following is an example of (2):

// crawlRequest.setSeedListId(seedListId); 

 

crawlRequest.setCrawlRegularExpression("^.+crawlexpression.com.*");

crawlRequest.setCrawlType(CrawlType.FAST);

crawlRequest.getMimeTypeList().add("text");

crawlRequest.setOutgoingLinkToCrawl(OutgoingLinkType.CRAWL_ALL_LINKS);

crawlRequest.setPreservingQueryStringWhenCrawling(true);

crawlRequest.setMaxNumberOfUrls(100);

crawlRequest.setMaxNumberUrlsPerPage(100);

crawlRequest.setDepthLevel(1);

job.setCrawlSetting(crawlRequest);

 

// Analysis Settings

AnalysisSetting analysisReq = new AnalysisSetting();

analysisReq.setAnalysisRegularExpression("");

 

// Note: the mime for the eighty app will automatically be set

analysisReq.setAnalysisMethod(AnalysisMethod.EIGHTY_APP);

 

// Note:  Please do not use setEightyAppId and instead use setEightyAppVersionId 

analysisReq.setEightyAppVersionId( [eightyAppVersionId]); //This is the id for the latest version of eighty app.  It can be found from Marketplace

                                                          //> Show mine link under 80aoo section.

 

// If the 80app has the following usage configuration: This 80app requires multiple lines of configuration data.". You may have a question of "How is this configuration

// information provided via API?

// The configuration should be provided using the data file.  This file can be uploaded just before the job is created and assigned to the job or it could have already been created and // used later.  

 

//Upload data file and get id

File f = new File("C://etc//80appJobConfiguration.txt");
int uploadedDataId= connector.uploadData(f, f.getName());

 

//If there is no need to upload data file than just set the data id.

//int uploadedDataId = 22;

           analysisReq.setDataId(uploadedDataId);

job.setAnalysisSetting(analysisReq);

 

 

// Result Settings

ResultSetting resultSetting = new ResultSetting();

//Built-in Keyword/Pattern Matching uses UNIQUE_AND_TOTAL_COUNT ResultType, optional of BOOLEAN_ARRAY and COUNT_ARRAY are available

resultSetting.setResultType(ResultType.COUNT_ARRAY);

resultSetting.setHasCrawlUrlsInResult(true);

resultSetting.setMaxResultFileSizeInMB(100);

job.setResultSetting(resultSetting);

 

return job;

     }

 

 

 

Download Job Results  

The following source code example illustrates how to check to see if a job run is completed and downloads the results of a job run through the 80legs API: 

 

package test.api; 

 

import java.io.IOException;

import java.util.ArrayList; 

 

import com.eightylegs.common.domain.APIProfile;

import com.eightylegs.common.domain.JobOverview;

import com.eightylegs.common.domain.JobRun;

import com.eightylegs.common.domain.JobRunStatusType;

import com.eightylegs.common.domain.JobStatusType;

import com.eightylegs.common.domain.RunResult;

import com.eightylegs.common.domain.WebServiceEnvironment;

import com.eightylegs.common.exception.EightyLegsParameterException;

import com.eightylegs.common.exception.EightyLegsXmlException;

import com.eightylegs.customer.EightyLegsRuntime;

import com.eightylegs.customer.api.IEightyLegsConnector;

import com.eightylegs.customer.api.JavaEightyLegsConnector;

import com.eightylegs.customer.exception.EightyLegsAPIException;

import com.eightylegs.customer.exception.EightyLegsConnectionException;

import com.eightylegs.customer.exception.EightyLegsException;

import com.eightylegs.customer.exception.EightyLegsFileNotFoundException;

import com.eightylegs.customer.exception.EightyLegsIOException;

 

 

public class TestCheckJobCompleteAndDownloadResultAPI {

 

 

      public static void main(String[] args) throws IOException {

  

            try {

                  

                 APIProfile profile = new APIProfile();
                 profile.setVersion("1.0");
                 profile.setApiToken("[Your API Token]");

                 IEightyLegsConnector connector = new JavaEightyLegsConnector(profile);

                 int jobId = 1601;
                 JobOverview jobOverview = connector.retrieveJobOverview(jobId);
                 if (jobOverview.getJobStatus() == JobStatusType.COMPLETED) {

                      // Retrieve the job runs
                      List<JobRun> runList = connector.retrieveJobRuns(jobId);
                      if (runList.size() > 0) {
                         for (JobRun run : runList) {

                             // Checks to see if run is completed
                             if (run.getRunStatus() == JobRunStatusType.COMPLETED) {
                                 List<RunResult> results = connector.retrieveRunResultsInfo(run.getId());
     
                                 // Downloads the result files only this method
                                 // returns the name of the downloaded files.

                                 List<String> resultMessages = connector.downloadResults(results, "C:\\etc\\downloadedResult\\");

                                 for (String resultMsg : resultMessages) {

                                     System.out.println(resultMsg);

                                 }

                                 // if you want to download the results and get the
                                 // url and bytes from the result,
                                 // use the following:
                                 // for(RunResult jobResult: results) {
                                 // HashMap<String,byte[]> resultMap = //
                                 // downloadAndReadCodeAnalysisResult(jobResult,C:\\downloadLoc\\)
                                 // }
                             }
                         }
                     }
                 } else {
                     System.out.println("Job is not completed.");
                 }

 

 

            } catch (EightyLegsAPIException e) {

                  e.printStackTrace();

            } catch (EightyLegsParameterException e) {

                  e.printStackTrace();

            } catch (EightyLegsXmlException e) {

                  e.printStackTrace();

            } catch (EightyLegsIOException e) {

                  e.printStackTrace();

            } catch (EightyLegsException e) {

                  e.printStackTrace();

            } catch (EightyLegsConnectionException e) {

                  e.printStackTrace();

            } catch (EightyLegsFileNotFoundException e) {

                  e.printStackTrace();

            }

      }

}

 

 Using Other API Operations

 

package test;

 

import java.io.IOException;

import java.io.UnsupportedEncodingException;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.HashMap;

 

import com.eightylegs.common.domain.APIProfile;

import com.eightylegs.common.domain.JobOverview;

import com.eightylegs.common.domain.JobRun;

import com.eightylegs.common.domain.JobSetting;

import com.eightylegs.common.domain.JobStatusType;

import com.eightylegs.common.domain.JobSummary;

import com.eightylegs.common.domain.RunResult;

import com.eightylegs.common.exception.EightyLegsParameterException;

import com.eightylegs.common.exception.EightyLegsXmlException;

import com.eightylegs.customer.api.IEightyLegsConnector;

import com.eightylegs.customer.api.JavaEightyLegsConnector;

import com.eightylegs.customer.exception.EightyLegsAPIException;

import com.eightylegs.customer.exception.EightyLegsConnectionException;

import com.eightylegs.customer.exception.EightyLegsException;

import com.eightylegs.customer.exception.EightyLegsFileNotFoundException;

import com.eightylegs.customer.exception.EightyLegsIOException;

 

public class TestJobAPIPub { 

 

// This method tests Sample Client Tracking Code JavaEightyLegsConnector functionality for Java Applications

 

public static void main(String[] args) throws IOException {

 

JobSetting jobSetting = null;

 

try {

 

// Create API Credentials

APIProfile profile = new APIProfile();

profile.setVersion("1.0");

profile.setApiToken("[API Token]");

 

IEightyLegsConnector connector = new JavaEightyLegsConnector(profile);

 

// Create a job object

jobSetting = JobGenerator.generateJob();

 

// Create a job

int jobIdToUse = connector.createJob(jobSetting);

 

// Retrieve a job's settings

JobSetting returnedJob = connector.retrieveJobSetting(jobIdToUse);

 

// Retrieve the current active jobs

ArrayList<JobSummary> jobs = connector.retrieveJobs(JobStatusType.CREATED);

 

// Copy an existing job to create a new job

int copiedJobId = connector.copyJob(jobIdToUse, "newName" + Calendar.getInstance().getTimeInMillis());

 

// Cancel a job

connector.cancelJob(jobIdToUse);

 

// Delete a job

connector.deleteJob(jobIdToUse);

 

// Retrieve a job's status

JobOverview jobOverview = connector.retrieveJobOverview(jobIdToUse);

 

// Retrieve information about a job run

ArrayList<JobRun> jobWithRunsInfo = connector.retrieveJobRuns(jobIdToUse);

 

// Retrieve the results from a job run

ArrayList<RunResult> results = connector.retrieveRunResultsInfo(225);

 

if (results.size() > 0) {

 

// Download the results

ArrayList<String> returnedMessage = connector.downloadResults(results, "C:\\test\\");

 

// Download results and return analysis results as <URL, result bytes>

HashMap<String, byte[]> downloadedAnalysisResultMap = connector.downloadAndReadCodeAnalysisResult(results.get(0),"C:\\test\\");

}

 

// Download the standard output for results for a job that was run in the sandbox

String resultGuid = "12277741-8e6b-4951-a74d-e17ada7b8a54";

String stdOutDownloadResult = connector.getStdoutForSandboxJob(resultGuid, null, "C:\\test\\");

 

} catch (UnsupportedEncodingException e) {

e.printStackTrace();

} catch (EightyLegsAPIException e) {

e.printStackTrace();

} catch (EightyLegsParameterException e) {

e.printStackTrace();

} catch (EightyLegsXmlException e) {

e.printStackTrace();

} catch (EightyLegsIOException e) {

e.printStackTrace();

} catch (EightyLegsException e) {

e.printStackTrace();

} catch (EightyLegsConnectionException e) {

e.printStackTrace();

} catch (EightyLegsFileNotFoundException e) {

e.printStackTrace();

} catch (Exception e) {

e.printStackTrace();

}

}

}

 

Delete Job After Downloading Results   

(Available in Version 1.0.9 and above)

 

The following source code example illustrates how to delete the jobs when downloading results.  After a job has been completed, it may take about 10 minues for the jobs to be posted.  We now have a isDonePostingResult flag which can be used to check if all the results are posted for the job.

 

     try {
            for (JobSummary jobSummary : connector.retrieveJobs(JobStatusType.COMPLETED)) {
                int jobId = jobSummary.getId();

                List<JobRun> runList = connector.retrieveJobRuns(jobId);
                List<RunResult> results = connector.retrieveRunResultsInfo(runList.get(0).getId());
                connector.downloadResults(results, "C:\\etc\\downloadedResult\\");
                if (runList.get(0).isDonePostingResult()) {
                    connector.deleteJob(jobId);
                }

            }
        } catch (EightyLegsCommonException e) {
            // TODO Auto-generated catch block
        ;    e.printStackTrace();
        }

 

Download new results for a Crawl Package

(Available in Version 1.0.9 and above)


        try {
            ArrayList<String> files = new ArrayList<String>();
            APIProfile profile = new APIProfile();
            profile.setVersion("1.0");
            profile.setApiToken("YOUR TOKEN");
            
            int CrawlPackageId = 0; //change this to the crawl package id that has been purchased.

            Map<integer,EightyApp> eightyAppMap = new HashMap<Integer, EightyApp>()


            int ppId = 0;
            IEightyLegsConnector connector = new JavaEightyLegsConnector(profile);
            List<RunResult> results = connector.retrieveRunResultsInfoForAllNewResultsForCrawlPackage(CrawlPackageId);
            for (RunResult r : results) {

                EightyApp app = eightyAppMap.get(r.getEightyAppVersionId());
                if (app  == null) {
                    app = connector.retrieveEightyAppByVersionId(r.getEightyAppVersionId());

                    eightyAppMap.put(r.getEightyAppVersionId(), app);

                }
                ppId = app.getPostProcessingCodeList().get(0).getId();
              
                if (r.getResultFileType().equals(ResultFileType.ANALYZED_URLS)) {
                    String filename = connector.downloadPostProcessedResults(r, null, "C:\\test\\",

                                        app.getLatestVersion().getId(), ppId, CrawlPackageId);
                    files.add(filename);
                }
            }
            

        } catch (Exception e) {
            e.printStackTrace();
        }
   

Download results for Crawl Package - Shows how to get Jobs and JobRuns

(Available in Version 1.0.9 and above)

     try {
            APIProfile profile = new APIProfile();
            profile.setVersion("1.0");
            profile.setApiToken("[YOUR TOKEN]");


            connector = new JavaEightyLegsConnector(profile);


            Map<integer,EightyApp> eightyAppMap = new HashMap<Integer, EightyApp>();

 

            // Get the list of crawl packages that have been purchased for your account.
            List<CrawlPackage> crawlPkgsList  = connector.retrieveAvailableCrawlPackagesByUser();

            if (crawlPkgsList.size() > 0) {
                for (CrawlPackage cpg : crawlPkgsList) {
                    System.out.println(cpg.getName());

                    // Get all the completed jobs for the crawl package.
                    List<JobSummary> jobs = connector.retrieveJobsForCrawlPackage(-1, cpg.getId());
                    for (JobSummary job : jobs) {

                        // Get all the runs for the job
                        List<JobRun> runs = connector.retrieveJobRunsForCrawlPackage(job.getId(), cpg.getId());
                        for (JobRun run : runs) {

                            // get all The results for the runs.
                            List<RunResult> results = run.getRunResults();
                            EightyApp app = null;
                            int ppId = 0;
                            for (RunResult r : results) {
                                

                                 EightyApp app = eightyAppMap.get(r.getEightyAppVersionId());
                                 if (app  == null) {
                                      app = connector.retrieveEightyAppByVersionId(r.getEightyAppVersionId());

                                      eightyAppMap.put(r.getEightyAppVersionId(), app);

                                 }
                                 ppId = app.getPostProcessingCodeList().get(0).getId();



                                // Download the postprocessed results only if
                                // the files have not been downloaded before.
                                if (r.getResultFileType().equals(ResultFileType.ANALYZED_URLS) && r.getNoOfTimesDownloaded() == 0) {
                                    String filename = connector.downloadPostProcessedResults(r, null, "C:\\test\\", app.getLatestVersion().getId(), ppId, cpg.getId());
                                    System.out.println(filename + " has been downloaded");
                                }
                            }
                        }
                    }
                }

            }

     } catch (Exception e) {
            e.printStackTrace();
     }

Comments (0)

You don't have permission to comment on this page.

PDF version