M src/main/java/com/civfanatics/storyarchiver/StoryArchiver.java +3 -2
@@ 156,15 156,16 @@ public class StoryArchiver {
WordpressConfig wpConfig = new WordpressConfig(Options.destinationFolder, resourcesFolder,
pageFolder, pageFolder + "index.html", stack);
+ wpConfig.setAsRoot();
- Writer fw = new OutputStreamWriter(new FileOutputStream("C:/temp/cfcArchive.html"), "UTF-8");
WordpressPage cfc = new WordpressPage("https://www.civfanatics.com", wpConfig);
stack.add(cfc);
//Runnable it. Use a Stack to track what remains to be downloaded.
while (!stack.empty()) {
- DownloadResource resource = stack.pop();
+ logger.info("Items remaining: " + stack.size());
+ DownloadResource resource = stack.pop();
DownloadRunnable runnable = new DownloadRunnable("", resource);
runnable.run();
}
M src/main/java/com/civfanatics/storyarchiver/wordpress/WordpressConfig.java +10 -0
@@ 14,6 14,8 @@ public class WordpressConfig {
private final String pageFolder;
private final String pageName;
private final Stack<DownloadResource> stackRef;
+ //Only pages marked as root will search for menu links
+ private boolean isRoot = false;
public WordpressConfig(String rootFolder, String resourcesFolder, String pageFolder, String pageName, Stack stackRef) {
this.rootFolder = rootFolder;
@@ 43,6 45,14 @@ public class WordpressConfig {
stackRef.add(page);
}
+ public void setAsRoot() {
+ this.isRoot = true;
+ }
+
+ public boolean isRoot() {
+ return isRoot;
+ }
+
public WordpressConfig cloneToChild(String childFolder, String childName) {
return new WordpressConfig(rootFolder, resourcesFolder, childFolder, childName, stackRef);
}
M src/main/java/com/civfanatics/storyarchiver/wordpress/WordpressPage.java +13 -4
@@ 10,10 10,12 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.ConnectException;
import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
+import java.util.logging.Level;
import java.util.logging.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@@ 46,6 48,7 @@ public class WordpressPage implements Do
Document doc = null;
int attempts = 0;
while (doc == null && attempts < 5) {
+ logger.log(Level.INFO, "Downloading " + url);
try {
doc = JSoupFetcher.getJSoupDocument(url);
}
@@ 97,6 100,16 @@ public class WordpressPage implements Do
URL ourURL = new URL(url);
String ourAuthority = ourURL.getAuthority();
+ if (wpConfig.isRoot()) {
+ traverseNavLinks(doc, ourURL, ourAuthority);
+ }
+
+ fw.write(header.outerHtml());
+ fw.write(primary.outerHtml());
+ fw.close();
+ }
+
+ private void traverseNavLinks(Document doc, URL ourURL, String ourAuthority) throws MalformedURLException {
Elements navLinks = doc.select("#site-navigation a[href]");
for (Element navLink : navLinks) {
System.out.println("Nav Link: " + navLink.attr("href") + " (" + navLink.text() + ")");
@@ 133,10 146,6 @@ public class WordpressPage implements Do
wpConfig.addToStack(childPage);
}
}
-
- fw.write(header.outerHtml());
- fw.write(primary.outerHtml());
- fw.close();
}
@Override