Ytukyg

So weird I get the results I need for the first 12 or so links then it crashes on the same link (AAA 090) every time. Im' not sure what the difference is between the links.

    import org.jsoup.*;

    import org.jsoup.helper.*;

    import org.jsoup.nodes.*;

    import org.jsoup.select.*;

    import org.w3c.dom.Document;



    import java.io.*; // Only needed if scraping a local File.



    public class Scraper {





        public Scraper() {



            org.jsoup.nodes.Document page = null;

            org.jsoup.nodes.Document prefix = null;

            org.jsoup.nodes.Document course = null;

            org.jsoup.nodes.Document cls = null;

            try {

                page = Jsoup.connect("https://erpdnssb.cccs.edu/PRODCCCS/ccns_pub_controller.p_command_processor?pi_course_status=A&pi_term_code=201920&pi_search_type=SB_PREFIX").get();



            // get the page title

            String title = page.title();

            System.out.println("title: " + title);



            // get all links in page

            Elements links = page.select("a[href]");

            for (Element link : links) {

                if(!link.text().equalsIgnoreCase("HELP")&&!link.text().equalsIgnoreCase("Exit"))

                { 

                    prefix = Jsoup.connect("https://erpdnssb.cccs.edu/PRODCCCS/" + link.attr("href")).get();

                    Elements links2 = prefix.select("a[href]");

                    for (Element link2 : links2) {

                        if(link2.text().matches("[A-Z]{3}"))

                        {

                            course = Jsoup.connect("https://erpdnssb.cccs.edu/PRODCCCS/" + link2.attr("href")).get();

                            Elements links3 = course.select("a[href]");

                            for (Element link3 : links3) {

                                if(link3.text().matches("[A-Z]{3} [0-9]{3}"))

                                {

                                    cls = Jsoup.connect("https://erpdnssb.cccs.edu/PRODCCCS/" + link3.attr("href")).get();

                                    Elements links4 = cls.getAllElements();





                                    //Elements courseNum = link4.select("body > div.pagebodydiv > table:nth-child(4) > tbody > tr:nth-child(1)");

                                    System.out.println("nhref = " + link3.attr("href") + "n" +  cls.text() + "n");

                                    System.out.println("link: " + cls.tagName());

                                    System.out.println("Course Number: " + link3.data().toString());



                                }

                            }

                        }

                    }

                }

            }

        }catch (IOException ioe) {

            ioe.printStackTrace();

        }

    }



    public static void main (String args) {



        new Scraper();

    }



}

org.jsoup.HttpStatusException: HTTP error fetching URL. Status=400, >URL=https://erpdnssb.cccs.edu/PRODCCCS/ccns_pub_controller.p_command_processor?>pi_search_type=SB_COURSE&_subj_code=AAA&pi_crse_numb=090&pi_archive_date=&pi_co>urse_status=A&pi_term_code=201920 at >org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:776 )at >org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:722) at >org.jsoup.helper.HttpConnection.execute(HttpConnection.java:306) at >org.jsoup.helper.HttpConnection.get(HttpConnection.java:295) at Scraper.(Scraper.java:42) at Scraper.main(Scraper.java:64)"

I think you are following the path "Why make it simple if it can be complicated?". I just have a look at that website you are trying to scrap. The site is well structured and organized. So no need for concatenating strings, using regex and do all the scraping in the constructor. I would sugest following steps:

1. Get the first page and select the table with the prefix links from A to Z


2. For each link from 1. get the page of the prefix and select the table with the course prefix links. For example for the prefix A to get the links from AAA to AVT


3. For each course prefix link from 2. get the page of the course prefix and select the table containing the course links. For example for AAA to get the links from AAA 010 to AAA 999

Your code could be something like :

import java.io.IOException;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;



public class JsoupTest {

    final static String homePage = "https://erpdnssb.cccs.edu/PRODCCCS/ccns_pub_controller.p_command_processor?"

                      + "pi_course_status=A&pi_term_code=201920&pi_search_type=SB_PREFIX";



    public static void main(String args) {

        try {

            Document page = Jsoup.connect(homePage).get();

            Elements prefixLinks = page.select(".pagebodydiv > table:nth-child(6) tbody tr td a");



            for(Element prefix : prefixLinks){

                Document prefixPage = Jsoup.connect(prefix.absUrl("href")).get();

                Elements coursePrefixLinks = prefixPage.select(".pagebodydiv > table:nth-child(6) tbody tr td a");



                for(Element coursePrefix : coursePrefixLinks){

                    Document coursePrefixPage = Jsoup.connect(coursePrefix.absUrl("href")).get();

                    Elements courseLinks = coursePrefixPage.select(".pagebodydiv > table:nth-child(6) tbody tr td a");                    



                    for(Element course : courseLinks){

                        Document coursePage = Jsoup.connect(course.absUrl("href")).get();

                        try{

                            String courseNr = coursePage.select("tr:contains(Course:)").first().text();  

                            String courseTitle = coursePage.select("tr:contains(Title:)").first().text(); 

                            System.out.println(courseNr +"n" + courseTitle + "n********************");

                        }

                        catch(NullPointerException np){

                            System.out.println("Broken link, Page not Found");

                        }

                    }

                }

            }



        } catch (IOException ex) {

            ex.printStackTrace();

        }

    }

}