diff --git a/build.gradle b/build.gradle index 58a83dc9..ee0e3a49 100644 --- a/build.gradle +++ b/build.gradle @@ -6,6 +6,7 @@ plugins { id "org.sonarqube" version "3.0" } + apply plugin: 'java' apply from: "gradle/git.gradle" apply plugin: 'checkstyle' @@ -17,7 +18,7 @@ compileJava.options.encoding = 'UTF-8' compileTestJava.options.encoding = 'UTF-8' ext.mainClass = 'moviescraper.doctord.Main' -version = getVersionCode() +//version = getVersionCode() repositories { mavenCentral() @@ -44,7 +45,7 @@ sourceSets { } dependencies { compile ( - 'commons-io:commons-io:2.7', + 'commons-io:commons-io:2.7', 'org.apache.commons:commons-lang3:3.11', 'commons-cli:commons-cli:1.4', 'commons-codec:commons-codec:1.14', @@ -53,7 +54,7 @@ dependencies { 'com.jgoodies:jgoodies-forms:1.9.0', 'org.imgscalr:imgscalr-lib:4.2', 'com.cedarsoftware:json-io:4.12.0', - 'org.jsoup:jsoup:1.7.3', + 'org.jsoup:jsoup:1.9.2', 'com.thoughtworks.xstream:xstream:1.4.12', 'org.json:json:20200518', 'org.hamcrest:hamcrest-core:2.2', @@ -76,7 +77,7 @@ jar { manifest { attributes( "Implementation-Title": "Gradle", - "Implementation-Version": version, + "Implementation-Version": archiveVersion, "Description": "JAV Movies scraper", "Main-Class" : mainClass ) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..62d4c053 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..5028f28f --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 00000000..fbd7c515 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000..a9f778a7 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/DmmParsingProfile.java b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/DmmParsingProfile.java old mode 100644 new mode 100755 index 8271165c..adac6306 --- a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/DmmParsingProfile.java +++ b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/DmmParsingProfile.java @@ -5,13 +5,15 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; +import java.sql.SQLOutput; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import moviescraper.doctord.controller.languagetranslation.JapaneseCharacter; import moviescraper.doctord.controller.languagetranslation.Language; import moviescraper.doctord.controller.languagetranslation.TranslateString; -import moviescraper.doctord.controller.siteparsingprofile.SecurityPassthrough; import moviescraper.doctord.controller.siteparsingprofile.SiteParsingProfile; import moviescraper.doctord.controller.siteparsingprofile.SiteParsingProfileJSON; import moviescraper.doctord.model.SearchResult; @@ -37,10 +39,10 @@ import moviescraper.doctord.model.dataitem.Year; import moviescraper.doctord.model.preferences.MoviescraperPreferences; +import moviescraper.doctord.scraper.UserAgent; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.text.WordUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -49,7 +51,7 @@ public class DmmParsingProfile extends SiteParsingProfile implements SpecificProfile { final static double dmmMaxRating = 5.00; - private boolean doGoogleTranslation; + private boolean doEnglishVersion; private boolean scrapeTrailers; @Override @@ -61,7 +63,7 @@ public List getScraperGroupNames() { public DmmParsingProfile() { super(); - doGoogleTranslation = (scrapingLanguage == Language.ENGLISH); + doEnglishVersion = (scrapingLanguage == Language.ENGLISH); // we can skip trailer scraping if user disables write trailer preference scrapeTrailers = MoviescraperPreferences.getInstance().getWriteTrailerToFile(); @@ -69,7 +71,7 @@ public DmmParsingProfile() { public DmmParsingProfile(Document document) { super(document); - doGoogleTranslation = (scrapingLanguage == Language.ENGLISH); + doEnglishVersion = (scrapingLanguage == Language.ENGLISH); } /** @@ -80,8 +82,8 @@ public DmmParsingProfile(Document document) { */ public DmmParsingProfile(boolean doGoogleTranslation) { super(); - this.doGoogleTranslation = doGoogleTranslation; - if (this.doGoogleTranslation == false) + this.doEnglishVersion = doGoogleTranslation; + if (this.doEnglishVersion == false) setScrapingLanguage(Language.JAPANESE); // we can skip trailer scraping if user disables write trailer preference @@ -90,52 +92,52 @@ public DmmParsingProfile(boolean doGoogleTranslation) { public DmmParsingProfile(boolean doGoogleTranslation, boolean scrapeTrailers) { super(); - this.doGoogleTranslation = doGoogleTranslation; - if (this.doGoogleTranslation == false) + this.doEnglishVersion = doGoogleTranslation; + if (this.doEnglishVersion == false) setScrapingLanguage(Language.JAPANESE); this.scrapeTrailers = scrapeTrailers; } public DmmParsingProfile(Document document, boolean doGoogleTranslation) { super(document); - this.doGoogleTranslation = doGoogleTranslation; - if (this.doGoogleTranslation == false) + this.doEnglishVersion = doGoogleTranslation; + if (this.doEnglishVersion == false) setScrapingLanguage(Language.JAPANESE); } @Override public Title scrapeTitle() { Element titleElement = document.select("[property=og:title]").first(); - // run a google translate on the japanese title - if (doGoogleTranslation) { - return new Title(TranslateString.translateStringJapaneseToEnglish(titleElement.attr("content").toString())); - } else { - return new Title(titleElement.attr("content").toString()); - } + ; + String title = titleElement.attr("content").toString(); + return new Title(title); } @Override public OriginalTitle scrapeOriginalTitle() { + if (doEnglishVersion) { + //English website does not have original Japanese title + return OriginalTitle.BLANK_ORIGINALTITLE; + } + Element titleElement = document.select("[property=og:title]").first(); - // leave the original title as the japanese title return new OriginalTitle(titleElement.attr("content").toString()); } @Override public SortTitle scrapeSortTitle() { - // we don't need any special sort title - that's usually something the - // user provides + // we don't need any special sort title - that's usually something the user provides return SortTitle.BLANK_SORTTITLE; } @Override public Set scrapeSet() { Element setElement = document.select("table.mg-b20 tr td a[href*=article=series/id=]").first(); + ; + if (setElement == null) return Set.BLANK_SET; - else if (doGoogleTranslation) { - return new Set(TranslateString.translateStringJapaneseToEnglish(setElement.text())); - } else + else return new Set(setElement.text()); } @@ -155,10 +157,15 @@ public Year scrapeYear() { @Override public ReleaseDate scrapeReleaseDate() { - Element releaseDateElement = document.select("table.mg-b20 tr td:contains(貸出開始日:) + td, table.mg-b20 tr td:contains(発売日:) + td, table.mg-b20 tr td:contains(商品発売日:) + td").first(); + Element releaseDateElement; + if (doEnglishVersion) { + releaseDateElement = document.select("table.mg-b20 tr td:contains(A sale date:) + td").first(); + } else { + releaseDateElement = document.select("table.mg-b20 tr td:contains(貸出開始日:) + td, table.mg-b20 tr td:contains(発売日:) + td, table.mg-b20 tr td:contains(商品発売日:) + td").first(); + } if (releaseDateElement != null) { String releaseDate = releaseDateElement.text(); - //we want to convert something like 2015/04/25 to 2015-04-25 + //we want to convert something like 2015/04/25 to 2015-04-25 releaseDate = StringUtils.replace(releaseDate, "/", "-"); return new ReleaseDate(releaseDate); } @@ -191,14 +198,16 @@ public Plot scrapePlot() { //dvd mode Element plotElement = document.select("p.mg-b20").first(); - if (plotElement == null || document.baseUri().contains("/digital/video") || document.baseUri().contains("/digital/nikkatsu")) { + if (plotElement == null || document.baseUri().contains("/digital/video")) { //video rental mode if it didnt find a match using above method plotElement = document.select("tbody .mg-b20.lh4").first(); } - if (doGoogleTranslation) { - return new Plot(TranslateString.translateStringJapaneseToEnglish(plotElement.text())); - } else - return new Plot(plotElement.text()); + + String plot = plotElement.text(); + + //remove special sale messages that occur after first "star" character + plot = plot.split("★", 2)[0]; + return new Plot(plot); } @Override @@ -215,7 +224,6 @@ public moviescraper.doctord.model.dataitem.Runtime scrapeRuntime() { runtime = runtimeElement.text().replaceAll("分", ""); } return new moviescraper.doctord.model.dataitem.Runtime(runtime); - } @Override @@ -278,7 +286,7 @@ public Thumb[] scrapePosters() { /** * Helper method for scrapePoster() and scapeFanart since this code is * virtually identical - * + * * @param doCrop * - if true, will only get the front cover as the initial poster * element; otherwise it uses the entire dvd case from DMM.co.jp @@ -301,9 +309,7 @@ private Thumb[] scrapePostersAndFanart(boolean doCrop, boolean scrapingExtraFana if (posterLink == null || posterLink.length() < 1) posterLink = postersElement.attr("abs:src"); try { - // for the poster, do a crop of the the right side of the dvd case image - //(which includes both cover art and back art) - // so we only get the cover + // for the poster, do a crop of the the right side of the dvd case image (which includes both cover art and back art) so we only get the cover if (doCrop && !scrapingExtraFanart) //use javCropCoverRoutine version of the new Thumb constructor to handle the cropping posters.add(new Thumb(posterLink, true)); @@ -348,15 +354,20 @@ public MPAARating scrapeMPAA() { @Override public ID scrapeID() { - Element idElement = document.select("td:containsOwn(品番:) ~ td").first(); + Element idElement; + if (doEnglishVersion) { + idElement = document.select("td:containsOwn(Movie Number:) ~ td").first(); + } else { + idElement = document.select("td:containsOwn(品番:) ~ td").first(); + } if (idElement != null) { String idElementText = idElement.text(); idElementText = fixUpIDFormatting(idElementText); return new ID(idElementText); - } - //This page didn't have an ID, so just put in a empty one - else + } else { + //This page didn't have an ID, so just put in a empty one return ID.BLANK_ID; + } } public static String fixUpIDFormatting(String idElementText) { @@ -375,7 +386,7 @@ public static String fixUpIDFormatting(String idElementText) { int firstNumberIndex = StringUtils.indexOfAny(idElementText, "0123456789"); idElementText = idElementText.substring(0, firstNumberIndex) + "-" + idElementText.substring(firstNumberIndex); - //remove extra zeros in case we get a 5 or 6 digit numerical part + //remove extra zeros in case we get a 5 or 6 digit numerical part //(For example ABC-00123 will become ABC-123) Pattern patternID = Pattern.compile("([0-9]*\\D+)(\\d{5,6})"); Matcher matcher = patternID.matcher(idElementText); @@ -395,6 +406,7 @@ public static String fixUpIDFormatting(String idElementText) { @Override public ArrayList scrapeGenres() { Elements genreElements = document.select("table.mg-b12 tr td a[href*=article=keyword/id=]"); + ArrayList genres = new ArrayList<>(genreElements.size()); for (Element genreElement : genreElements) { // get the link so we can examine the id and do some sanity cleanup @@ -403,25 +415,21 @@ public ArrayList scrapeGenres() { String href = genreElement.attr("abs:href"); String genreID = genreElement.attr("abs:href").substring(href.indexOf("id=") + 3, href.length() - 1); if (acceptGenreID(genreID)) { - if (doGoogleTranslation == false) { + if (!doEnglishVersion) { genres.add(new Genre(genreElement.text())); } else { String potentialBetterTranslation = betterGenreTranslation(genreElement.text(), genreID); - // we didn't know of anything hand picked for genres, just use - // google translate if (potentialBetterTranslation.equals("")) { - genres.add(new Genre(TranslateString.translateStringJapaneseToEnglish(genreElement.text()))); - } - // Cool, we got something we want to use instead for our genre, - // let's use that - else { + // use genre found on site + genres.add(new Genre(genreElement.text())); + } else { + // use our genre name genres.add(new Genre(potentialBetterTranslation)); } } } } - // System.out.println("genres" + genreElements); return genres; } @@ -502,7 +510,9 @@ private String betterActressTranslation(String text, String actressID) { break; } - + if (betterActressTranslatedString.equals("")) { + return text; + } return betterActressTranslatedString; } @@ -526,77 +536,66 @@ private boolean acceptGenreID(String genreID) { @Override public ArrayList scrapeActors() { + + boolean doDmmActressScraping = MoviescraperPreferences.getInstance().getScrapeDmmActressPref(); + if (!doDmmActressScraping) { + System.out.println("DMM Scraper: Skipping actress scraping. (see Scraper's Setting)"); + return (new ArrayList<>()); + } + // scrape all the actress IDs Elements actressIDElements = document.select("span#performer a[href*=article=actress/id=]"); + + if (actressIDElements.size() < 1) { + System.out.println("DMM Scraper: No actress found."); + return (new ArrayList<>()); + } + + //setup cookies and user agent for Jsoup + Map cookies = new HashMap(); + cookies.put("age_check_done", "1"); + + String actressPageURL; + if (doEnglishVersion) { + actressPageURL = "https://actress.dmm.co.jp/en/-/detail/=/actress_id="; + + // set cookies for EN version + cookies.put("ckcy", "2"); + cookies.put("cklg", "en"); + } else { + actressPageURL = "https://actress.dmm.co.jp/-/detail/=/actress_id="; + } ArrayList actorList = new ArrayList<>(actressIDElements.size()); + + //there maybe multiple actress. let's process each actress. for (Element actressIDLink : actressIDElements) { + String actressName = actressIDLink.text(); String actressIDHref = actressIDLink.attr("abs:href"); - String actressNameKanji = actressIDLink.text(); String actressID = actressIDHref.substring(actressIDHref.indexOf("id=") + 3, actressIDHref.length() - 1); - String actressPageURL = "https://actress.dmm.co.jp/-/detail/=/actress_id=" + actressID + "/"; + + System.out.println("DMM Scraper: getting actresses from " + actressPageURL + actressID + "/"); try { - Document actressPage = Jsoup.connect(actressPageURL).timeout(SiteParsingProfile.CONNECTION_TIMEOUT_VALUE).get(); - Element actressNameElement = actressPage.select("td.t1 h1").first(); + Document actressPage = Jsoup.connect(actressPageURL + actressID + "/") + //.header("Cache-Control", "no-store").header("Connection", "close") + .cookies(cookies).userAgent(UserAgent.getUserAgent(0)).ignoreHttpErrors(true).timeout(CONNECTION_TIMEOUT_VALUE).post(); + Element actressThumbnailElement = actressPage.select("tr.area-av30.top td img").first(); String actressThumbnailPath = actressThumbnailElement.attr("abs:src"); - //Sometimes the translation service from google gives us weird engrish instead of a name, so let's compare it to the thumbnail file name for the image as a sanity check - //if the names aren't close enough, we'll use the thumbnail name - //many times the thumbnail name is off by a letter or two or has a number in it, which is why we just don't use this all the time... - String actressNameFromThumbnailPath = actressThumbnailPath.substring(actressThumbnailPath.lastIndexOf('/') + 1, actressThumbnailPath.lastIndexOf('.')); - - //To do a proper comparison using Lev distance, let's fix case, make first name appear first get rid of numbers - actressNameFromThumbnailPath = actressNameFromThumbnailPath.replaceAll("[0-9]", ""); - actressNameFromThumbnailPath = actressNameFromThumbnailPath.replaceAll("_", " "); - actressNameFromThumbnailPath = WordUtils.capitalize(actressNameFromThumbnailPath); - actressNameFromThumbnailPath = StringUtils.reverseDelimited(actressNameFromThumbnailPath, ' '); - - // The actor's name is easier to google translate if we get the - // hiragana form of it. - // The hiragana form of it is between a '(' and a ')' (These are - // not parens but some japanese version of parens) - String actressNameHiragana = actressNameElement.text().substring(actressNameElement.text().indexOf('(') + 1, actressNameElement.text().indexOf(')')); - // maybe we know in advance the translation system will be junk, - // so we check our manual override of people we know it will get - // the name wrong on - String actressNameEnglish = betterActressTranslation(actressNameHiragana, actressID); - boolean didWeManuallyOverrideActress = false; - if (actressNameEnglish.equals("") && doGoogleTranslation) { - actressNameEnglish = TranslateString.translateJapanesePersonNameToRomaji(actressNameHiragana); - } else - didWeManuallyOverrideActress = true; - - //use the difference between the two strings to determine which is the better one. The google translate shouldn't be that many characters away from the thumbnail name, or it's garbage - //unless the thumbnail name was the generic "Nowprinting" one, in which case use the google translate - if (!actressNameFromThumbnailPath.equals("Nowprinting")) { - int LevenshteinDistance = StringUtils.getLevenshteinDistance(actressNameEnglish, actressNameFromThumbnailPath); - if (LevenshteinDistance > 3 && !didWeManuallyOverrideActress) { - //System.out.println("(We found a junk result from google translate, swapping over to cleaned up thumbnail name"); - //System.out.println("Google translate's version of our name: " + actressNameEnglish + " Thumbnail name of person: " + actressNameFromThumbnailPath + " Lev Distance: " + LevenshteinDistance + ")"); - actressNameEnglish = actressNameFromThumbnailPath; - } + + if (doEnglishVersion) { + actressName = betterActressTranslation(actressName, actressID); } //Sometimes DMM lists a fake under the Name "Main". It's weird and it's not a real person, so just ignore it. - if (!actressNameEnglish.equals("Main")) { - - if (doGoogleTranslation) { - if (!actressThumbnailPath.contains("nowprinting.gif")) { - actorList.add(new Actor(actressNameEnglish, "", new Thumb(actressThumbnailPath))); - } else { - actorList.add(new Actor(actressNameEnglish, "", null)); - } - + if (!actressName.equals("Main")) { + if (!actressThumbnailPath.contains("nowprinting.gif")) { + actorList.add(new Actor(actressName, "", new Thumb(actressThumbnailPath))); } else { - if (!actressThumbnailPath.contains("nowprinting.gif")) { - actorList.add(new Actor(actressNameKanji, "", new Thumb(actressThumbnailPath))); - } else { - actorList.add(new Actor(actressNameKanji, "", null)); - } + actorList.add(new Actor(actressName, "", null)); } } - } catch (SocketTimeoutException e) { - System.err.println("Cannot download from " + actressPageURL.toString() + ": Socket timed out: " + e.getLocalizedMessage()); + System.err.println("DMM Scraper: Cannot download from " + actressPageURL.toString() + ": Socket timed out: " + e.getLocalizedMessage()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -604,12 +603,13 @@ public ArrayList scrapeActors() { } //Get actors that are just a "Name" and have no page of their own (common on some web releases) + //TODO Z Refactor "name only actors" to *not* use English translator service. Need examples of these. Elements nameOnlyActors = document.select("table.mg-b20 tr td:contains(�??�?:) + td"); for (Element currentNameOnlyActor : nameOnlyActors) { String actorName = currentNameOnlyActor.text().trim(); //for some reason, they sometimes list the age of the person after their name, so let's get rid of that actorName = actorName.replaceFirst("\\([0-9]{2}\\)", ""); - if (doGoogleTranslation) + if (doEnglishVersion) actorName = TranslateString.translateJapanesePersonNameToRomaji(actorName); actorList.add(new Actor(actorName, "", null)); } @@ -621,23 +621,28 @@ public ArrayList scrapeActors() { public ArrayList scrapeDirectors() { ArrayList directors = new ArrayList<>(); Element directorElement = document.select("table.mg-b20 tr td a[href*=article=director/id=]").first(); + if (directorElement != null && directorElement.hasText()) { - if (doGoogleTranslation) - directors.add(new Director(TranslateString.translateStringJapaneseToEnglish(directorElement.text()), null)); - else - directors.add(new Director(directorElement.text(), null)); + directors.add(new Director(directorElement.text(), null)); + System.out.println("DMM Scraper: Directors --> " + directorElement.text()); + } else { + System.out.println("DMM Scraper: No director found."); } return directors; } @Override public Studio scrapeStudio() { - Element studioElement = document.select("td:containsOwn(メーカー:) ~ td").first(); + Element studioElement; + + if (doEnglishVersion) { + studioElement = document.select("td:containsOwn(Studios:) ~ td").first(); + } else { + studioElement = document.select("td:containsOwn(メーカー:) ~ td").first(); + } + if (studioElement != null) { - if (doGoogleTranslation) - return new Studio(TranslateString.translateStringJapaneseToEnglish(studioElement.text())); - else - return new Studio(studioElement.text()); + return new Studio(studioElement.text()); } else return Studio.BLANK_STUDIO; } @@ -646,12 +651,19 @@ public Studio scrapeStudio() { public String createSearchString(File file) { scrapedMovieFile = file; String fileNameNoExtension = findIDTagFromFile(file, isFirstWordOfFileIsID()); - //System.out.println("fileNameNoExtension in DMM: " + fileNameNoExtension); + URLCodec codec = new URLCodec(); try { String fileNameURLEncoded = codec.encode(fileNameNoExtension); - //System.out.println("FileNameUrlencode = " + fileNameURLEncoded); - return "https://www.dmm.co.jp/search/=/searchstr=" + fileNameURLEncoded + "/"; + + String searchString; + if (doEnglishVersion) { + searchString = "https://www.dmm.co.jp/en/search/=/searchstr=" + fileNameURLEncoded + "/"; + } else { + searchString = "https://www.dmm.co.jp/search/=/searchstr=" + fileNameURLEncoded + "/"; + } + System.out.println("DMM Scraper: Search string --> " + searchString); + return searchString; } catch (Exception e) { e.printStackTrace(); } @@ -661,13 +673,24 @@ public String createSearchString(File file) { /** * returns a String[] filled in with urls of each of the possible movies * found on the page returned from createSearchString - * + * * @throws IOException */ @Override public SearchResult[] getSearchResults(String searchString) throws IOException { boolean firstPageScraping = true; - Document searchResultsPage = Jsoup.connect(searchString).timeout(CONNECTION_TIMEOUT_VALUE).get(); + + Document searchResultsPage = Jsoup.connect(searchString) + //.header("Cache-Control", "no-store").header("Connection", "close") + .userAgent(UserAgent.getUserAgent(0)).ignoreHttpErrors(true).timeout(CONNECTION_TIMEOUT_VALUE).post(); + + //did we get the no-result page? + Element noResult = searchResultsPage.select("div.d-rst.whole.search-noresult").first(); + if (noResult != null) { + System.out.println("DMM Scraper: No Result --> " + noResult.select("p.red").first().text()); + return null; + } + Element nextPageLink = searchResultsPage.select("div.list-capt div.list-boxcaptside.list-boxpagenation ul li:not(.terminal) a").last(); ArrayList searchResults = new ArrayList<>(); ArrayList pagesVisited = new ArrayList<>(); @@ -682,7 +705,7 @@ public SearchResult[] getSearchResults(String searchString) throws IOException { //so for now I'm making each its own variable and looping through and adding in all the elements seperately Elements dvdLinks = searchResultsPage.select("p.tmb a[href*=/mono/dvd/"); Elements rentalElements = searchResultsPage.select("p.tmb a[href*=/rental/ppr/"); - Elements digitalElements = searchResultsPage.select("p.tmb a[href*=/digital/videoa/], p.tmb a[href*=/digital/videoc/], p.tmb a[href*=/digital/nikkatsu/]"); + Elements digitalElements = searchResultsPage.select("p.tmb a[href*=/digital/videoa/], p.tmb a[href*=/digital/videoc/]"); //get /mono/dvd links for (int i = 0; i < dvdLinks.size(); i++) { @@ -723,7 +746,9 @@ public SearchResult[] getSearchResults(String searchString) throws IOException { //TODO this is really not the cleanest way of doing this - I can probably find some way to make the selector not send me in a loop //of pages, but this will work for now if (nextPageLink != null && !pagesVisited.contains(nextPageURL)) - searchResultsPage = Jsoup.connect(nextPageURL).get(); + searchResultsPage = Jsoup.connect(nextPageURL) + //.header("Cache-Control", "no-store").header("Connection", "close") + .userAgent(UserAgent.getUserAgent(0)).ignoreHttpErrors(true).timeout(CONNECTION_TIMEOUT_VALUE).post(); else break; @@ -775,18 +800,78 @@ public Document downloadDocument(SearchResult searchResult) { if (searchResult.isJSONSearchResult()) return SiteParsingProfileJSON.getDocument(searchResult.getUrlPath()); else { - - //setup cookie to bypass age check on DMM site Map cookies = new HashMap(); - cookies.put("age_check_done", "1"); + cookies.put("age_check_done", "1"); //setup cookie to bypass age check on DMM site + + String searchUrl; + if (doEnglishVersion) { + //setup cookies for getting English version + cookies.put("ckcy", "2"); + cookies.put("cklg", "en"); + searchUrl = searchResult.getUrlPath().replace("dmm.co.jp/", "dmm.co.jp/en/"); + + //Append a dummy URL parameter to see if it helps by pass server cache + //searchUrl += "&ymmud=" + System.currentTimeMillis(); + + System.out.println("DMM Scraper: getting EN version at " + searchUrl); + } else { + searchUrl = searchResult.getUrlPath(); + System.out.println("DMM Scraper: getting JP version at " + searchUrl); + } + + Document document = Jsoup.connect(searchUrl).cookies(cookies) + //.header("Cache-Control", "no-store").header("Connection", "close") + .userAgent(UserAgent.getUserAgent(0)).ignoreHttpErrors(true).timeout(CONNECTION_TIMEOUT_VALUE).post(); + + String title = document.select("[property=og:title]").first().attr("content").toString(); + String plot = document.select("p.mg-b20").first().text(); + System.out.println("DMM Scraper: Title --> " + title); + System.out.println("DMM Scraper: Plot --> " + plot); + + if (doEnglishVersion) { + //Sometimes we get Japanese results even though our request is for English. + //Probably due to webserver caching. Our 2nd request might be too quick. + if (this.hasJapanese(title)) { + //one more attempt to scrape EN version after a small time delay + System.out.println("DMM Scraper: Failed at getting EN version. Result is JP. Title --> " + title); + System.out.println("DMM Scraper: waiting 5 seconds before attempting to get EN version again..."); + TimeUnit.SECONDS.sleep(5); + + System.out.println("DMM Scraper: getting EN version at " + searchUrl); + document = Jsoup.connect(searchUrl).cookies(cookies) + //.header("Cache-Control", "no-store").header("Connection", "close") + .userAgent(UserAgent.getUserAgent(0)).ignoreHttpErrors(true).timeout(CONNECTION_TIMEOUT_VALUE).post(); + + title = document.select("[property=og:title]").first().attr("content").toString(); + plot = document.select("p.mg-b20").first().text(); + System.out.println("DMM Scraper: Title --> " + title); + System.out.println("DMM Scraper: Plot --> " + plot); + } + } - Document doc = Jsoup.connect(searchResult.getUrlPath()).cookies(cookies).userAgent("Mozilla").ignoreHttpErrors(true).timeout(CONNECTION_TIMEOUT_VALUE).get(); - return doc; + return document; } - } catch (IOException e) { + } catch (IOException | InterruptedException e) { e.printStackTrace(); } return null; } -} + /* + * Check for Japanese characters + */ + private static boolean hasJapanese(CharSequence charSequence) { + boolean hasJapanese = false; + for (char c : charSequence.toString().toCharArray()) { + if (Character.UnicodeBlock.of(c) == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.HIRAGANA + || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.KATAKANA || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS + || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS || Character.UnicodeBlock.of(c) == Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION) { + hasJapanese = true; + break; + } + } + + return hasJapanese; + } + +} \ No newline at end of file diff --git a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/R18ParsingProfile.java b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/R18ParsingProfile.java index 3cc552f4..7a407a37 100644 --- a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/R18ParsingProfile.java +++ b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/R18ParsingProfile.java @@ -43,6 +43,7 @@ import moviescraper.doctord.model.dataitem.Trailer; import moviescraper.doctord.model.dataitem.Votes; import moviescraper.doctord.model.dataitem.Year; +import moviescraper.doctord.scraper.UserAgent; public class R18ParsingProfile extends SiteParsingProfile implements SpecificProfile { @@ -399,7 +400,7 @@ private SearchResult[] searchResultOnR18(String searchWord) { searchWordURLEncoded = codec.encode(searchWord); String searchPattern = "http://www.r18.com/common/search/floor=movies/searchword=" + searchWordURLEncoded + "/"; System.out.println("Searching on R18 with this URL:" + searchPattern); - Document searchResultsPage = Jsoup.connect(searchPattern).timeout(SiteParsingProfile.CONNECTION_TIMEOUT_VALUE).get(); + Document searchResultsPage = Jsoup.connect(searchPattern).userAgent(UserAgent.getUserAgent(0)).timeout(SiteParsingProfile.CONNECTION_TIMEOUT_VALUE).get(); Elements moviesFound = searchResultsPage.select(".cmn-list-product01 li"); if (moviesFound != null && moviesFound.size() > 0) { SearchResult[] foundResults = new SearchResult[moviesFound.size()]; diff --git a/src/main/java/moviescraper/doctord/controller/xmlserialization/KodiXmlMovieBean.java b/src/main/java/moviescraper/doctord/controller/xmlserialization/KodiXmlMovieBean.java index d53ed935..c095fb11 100644 --- a/src/main/java/moviescraper/doctord/controller/xmlserialization/KodiXmlMovieBean.java +++ b/src/main/java/moviescraper/doctord/controller/xmlserialization/KodiXmlMovieBean.java @@ -44,6 +44,9 @@ public class KodiXmlMovieBean { public static KodiXmlMovieBean makeFromXML(String xml) { XStream xstream = KodiXmlMovieBean.getXMLSerializer(); + xstream.allowTypesByWildcard(new String[] { + "moviescraper.doctord.**" + }); xstream.ignoreUnknownElements(); try { KodiXmlMovieBean beanToReturn = (KodiXmlMovieBean) xstream.fromXML(xml); diff --git a/src/main/java/moviescraper/doctord/model/Movie.java b/src/main/java/moviescraper/doctord/model/Movie.java index 5987b079..f3fd9ba6 100644 --- a/src/main/java/moviescraper/doctord/model/Movie.java +++ b/src/main/java/moviescraper/doctord/model/Movie.java @@ -13,6 +13,8 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.imageio.IIOImage; import javax.imageio.ImageIO; @@ -83,8 +85,8 @@ public class Movie { private String fileName; public Movie(ArrayList actors, ArrayList directors, Thumb[] fanart, Thumb[] extraFanart, ArrayList genres, ArrayList tags, ID id, MPAARating mpaa, - OriginalTitle originalTitle, Outline outline, Plot plot, Thumb[] posters, Rating rating, ReleaseDate releaseDate, Runtime runtime, Set set, SortTitle sortTitle, Studio studio, - Tagline tagline, Title title, Top250 top250, Trailer trailer, Votes votes, Year year) { + OriginalTitle originalTitle, Outline outline, Plot plot, Thumb[] posters, Rating rating, ReleaseDate releaseDate, Runtime runtime, Set set, SortTitle sortTitle, Studio studio, + Tagline tagline, Title title, Top250 top250, Trailer trailer, Votes votes, Year year) { super(); this.actors = actors; this.directors = directors; @@ -212,7 +214,7 @@ private void setDataItemSourceOnThumbs(Thumb[] thumbs, DataItemSource dataItemSo /** * Create a movie by reading in a values from a nfo file created by previously scraping the movie and then writing the metadata out to the file - * + * * @param nfoFile * @throws IOException */ @@ -412,9 +414,9 @@ public void setYear(Year year) { @Override public String toString() { return "Movie [title=" + title + ", originalTitle=" + originalTitle + ", sortTitle=" + sortTitle + ", set=" + set + ", rating=" + rating + ", year=" + year + ", top250=" + top250 - + ", trailer = " + trailer + ", votes=" + votes + ", outline=" + outline + ", plot=" + plot + ", tagline=" + tagline + ", studio=" + studio + "releaseDate=" + releaseDate - + ", runtime=" + runtime + ", posters=" + Arrays.toString(posters) + ", fanart=" + Arrays.toString(fanart) + ", extrafanart = " + Arrays.toString(extraFanart) + ", mpaa=" + mpaa - + ", id=" + id + ", genres=" + genres + ", tags=" + tags + ", actors=" + actors + ", directors=" + directors + "]"; + + ", trailer = " + trailer + ", votes=" + votes + ", outline=" + outline + ", plot=" + plot + ", tagline=" + tagline + ", studio=" + studio + "releaseDate=" + releaseDate + + ", runtime=" + runtime + ", posters=" + Arrays.toString(posters) + ", fanart=" + Arrays.toString(fanart) + ", extrafanart = " + Arrays.toString(extraFanart) + ", mpaa=" + mpaa + + ", id=" + id + ", genres=" + genres + ", tags=" + tags + ", actors=" + actors + ", directors=" + directors + "]"; } public String toXML() { @@ -440,7 +442,7 @@ public void writeExtraFanart(File directoryMovieIsIn) throws IOException { } public void writeToFile(File nfoFile, File posterFile, File fanartFile, File currentlySelectedFolderJpgFile, File targetFolderForExtraFanartFolderAndActorFolder, File trailerFile, - MoviescraperPreferences preferences) throws IOException { + MoviescraperPreferences preferences) throws IOException { // Output the movie to XML using XStream and a proxy class to // translate things to a format that Kodi expects @@ -471,7 +473,7 @@ public void writeToFile(File nfoFile, File posterFile, File fanartFile, File cur // save the first poster out // maybe we did some clipping, so we're going to have to reencode it if (this.getPosters().length > 0 && (writePoster || createFolderJpgEnabledPreference) - && ((posterFile.exists() == writePosterIfAlreadyExists) || (!posterFile.exists() || (createFolderJpgEnabledPreference)))) { + && ((posterFile.exists() == writePosterIfAlreadyExists) || (!posterFile.exists() || (createFolderJpgEnabledPreference)))) { if (posterToSaveToDisk != null && (posterToSaveToDisk.isModified() || createFolderJpgEnabledPreference || !posterFile.exists() || writePosterIfAlreadyExists)) { //reencode the jpg since we probably did a resize Iterator iter = ImageIO.getImageWritersByFormatName("jpeg"); @@ -657,7 +659,7 @@ public static String getFileNameOfExtraFanartFolderName(File selectedValue) { } public static String getFileNameOfTrailer(File selectedValue) { - //sometimes the trailer has a different extension + //sometimes the trailer has a different extension //than the movie so we will try to brute force a find by trying all movie name extensions for (String extension : MovieFilenameFilter.acceptedMovieExtensions) { String potentialTrailer = tryToFindActualTrailerHelper(selectedValue, "." + extension); @@ -669,7 +671,7 @@ public static String getFileNameOfTrailer(File selectedValue) { /** * Checks for the given file a trailer file exists for it for the given file name extension - * + * * @param selectedValue - base file name of movie or nfo * @param extension - the file name extension we are checking * @return - the path to the file if it found the trailer, otherwise null @@ -730,35 +732,6 @@ public boolean accept(File directory, String fileName) { } } - /* - * private String [] searchResultsHelperForScrapeMovie(File movieFile, SiteParsingProfile siteToParseFrom) - * { - * String [] searchResults = siteToParseFrom.getSearchResults(searchString); - * int levDistanceOfCurrentMatch = 999999; // just some super high number - * String idFromMovieFile = SiteParsingProfile.findIDTagFromFile(movieFile); - * - * //loop through search results and see if URL happens to contain ID number in the URL. This will improve accuracy! - * for (int i = 0; i < searchResults.length; i++) - * { - * String urltoMatch = searchResults[i].toLowerCase(); - * String idFromMovieFileToMatch = idFromMovieFile.toLowerCase().replaceAll("-", ""); - * //System.out.println("Comparing " + searchResults[i].toLowerCase() + " to " + idFromMovieFile.toLowerCase().replaceAll("-", "")); - * if (urltoMatch.contains(idFromMovieFileToMatch)) - * { - * //let's do some fuzzy logic searching to try to get the "best" match in case we got some that are pretty close - * //and update the variables accordingly so we know what our best match so far is - * int candidateLevDistanceOfCurrentMatch = StringUtils.getLevenshteinDistance(urltoMatch.toLowerCase(), idFromMovieFileToMatch); - * if (candidateLevDistanceOfCurrentMatch < levDistanceOfCurrentMatch) - * { - * levDistanceOfCurrentMatch = candidateLevDistanceOfCurrentMatch; - * searchResultNumberToUse = i; - * } - * } - * } - * return searchResults; - * } - */ - //Version that allows us to update the GUI while scraping public static Movie scrapeMovie(File movieFile, SiteParsingProfile siteToParseFrom, String urlToScrapeFromDMM, boolean useURLtoScrapeFrom) throws IOException { @@ -767,25 +740,45 @@ public static Movie scrapeMovie(File movieFile, SiteParsingProfile siteToParseFr return null; String searchString = siteToParseFrom.createSearchString(movieFile); SearchResult[] searchResults = null; - int searchResultNumberToUse = 0; + int searchResultNumberToUse = -1; //no URL was passed in so we gotta figure it ourselves if (!useURLtoScrapeFrom) { searchResults = siteToParseFrom.getSearchResults(searchString); int levDistanceOfCurrentMatch = 999999; // just some super high number String idFromMovieFile = SiteParsingProfile.findIDTagFromFile(movieFile, siteToParseFrom.isFirstWordOfFileIsID()); - //loop through search results and see if URL happens to contain ID number in the URL. This will improve accuracy! + /* + * We will loop through the search results and compare the Label field for match movie ID match. If we + * cannot find match, we will compare to URL and hope to find movie ID match there. + */ + //Loop through search results and compare Label field for match + String pattern = "(?<=_|\\b)" + idFromMovieFile.toUpperCase() + "(?=_|\\b)"; + Pattern p = Pattern.compile(pattern); for (int i = 0; i < searchResults.length; i++) { - String urltoMatch = searchResults[i].getUrlPath().toLowerCase(); - String idFromMovieFileToMatch = idFromMovieFile.toLowerCase().replaceAll("-", ""); - //System.out.println("Comparing " + searchResults[i].toLowerCase() + " to " + idFromMovieFile.toLowerCase().replaceAll("-", "")); - if (urltoMatch.contains(idFromMovieFileToMatch)) { - //let's do some fuzzy logic searching to try to get the "best" match in case we got some that are pretty close - //and update the variables accordingly so we know what our best match so far is - int candidateLevDistanceOfCurrentMatch = StringUtils.getLevenshteinDistance(urltoMatch.toLowerCase(), idFromMovieFileToMatch); - if (candidateLevDistanceOfCurrentMatch < levDistanceOfCurrentMatch) { - levDistanceOfCurrentMatch = candidateLevDistanceOfCurrentMatch; - searchResultNumberToUse = i; + Matcher match = p.matcher(searchResults[i].getLabel().toUpperCase()); + if (match.find() == true) { + searchResultNumberToUse = i; + break; + } + } + + if (searchResultNumberToUse == -1) { + //We did not find match using Label field so let's + //loop through search results again and see if URL happens to contain ID number in the URL. + searchResultNumberToUse = 0; + for (int i = 0; i < searchResults.length; i++) { + String urltoMatch = searchResults[i].getUrlPath().toLowerCase(); + String idFromMovieFileToMatch = idFromMovieFile.toLowerCase().replaceAll("-", ""); + + //System.out.println("Comparing " + searchResults[i].toLowerCase() + " to " + idFromMovieFile.toLowerCase().replaceAll("-", "")); + if (urltoMatch.contains(idFromMovieFileToMatch)) { + //let's do some fuzzy logic searching to try to get the "best" match in case we got some that are pretty close + //and update the variables accordingly so we know what our best match so far is + int candidateLevDistanceOfCurrentMatch = StringUtils.getLevenshteinDistance(urltoMatch.toLowerCase(), idFromMovieFileToMatch); + if (candidateLevDistanceOfCurrentMatch < levDistanceOfCurrentMatch) { + levDistanceOfCurrentMatch = candidateLevDistanceOfCurrentMatch; + searchResultNumberToUse = i; + } } } } @@ -902,7 +895,7 @@ public static Movie getEmptyMovie() { Year year = Year.BLANK_YEAR; return new Movie(actors, directors, fanart, extraFanart, genres, tags, id, mpaa, originalTitle, outline, plot, posters, rating, releaseDate, runtime, set, sortTitle, studio, tagline, title, - top250, trailer, votes, year); + top250, trailer, votes, year); } public String getFileName() { @@ -924,7 +917,7 @@ public void setReleaseDate(ReleaseDate releaseDate) { /** * remove the item from the picked from the existing poster list and put it at * the front of the list. if the movie does not contain the poster, no change will be made - * + * * @param posterToGoToFront - poster to put in front */ public void moveExistingPosterToFront(Thumb posterToGoToFront) { @@ -943,7 +936,7 @@ public void moveExistingPosterToFront(Thumb posterToGoToFront) { /** * remove the item from the picked from the existing fanart list and put it at * the front of the list. if the movie does not contain the fanart, no change will be made - * + * * @param fanartToGoToFront - fanart to put in front */ public void moveExistingFanartToFront(Thumb fanartToGoToFront) { diff --git a/src/main/java/moviescraper/doctord/model/preferences/MoviescraperPreferences.java b/src/main/java/moviescraper/doctord/model/preferences/MoviescraperPreferences.java index 4d024d94..140a7fce 100644 --- a/src/main/java/moviescraper/doctord/model/preferences/MoviescraperPreferences.java +++ b/src/main/java/moviescraper/doctord/model/preferences/MoviescraperPreferences.java @@ -34,7 +34,8 @@ enum Key implements Settings.Key { frequentlyUsedTags, //Used in tag editing to store user's list of frequently used tags to aid in quickly adding tags to a movie writeThumbTagsForPosterAndFanartToNfo, //Whether to write the tag into the nfo, userAgent, //UserAgent to use - cookieJar //UserAgent to use + cookieJar, //UserAgent to use + scrapeDmmActressPref //option to scrape actress when DMM scraper is used ; @Override @@ -171,6 +172,14 @@ public void setScrapeInJapanese(Boolean preferenceValue) { setBooleanValue(Key.scrapeInJapanese, preferenceValue); } + public Boolean getScrapeDmmActressPref() { + return getBooleanValue(Key.scrapeDmmActressPref, Boolean.TRUE); + } + + public void setScrapeDmmActressPref(Boolean preferenceValue) { + setBooleanValue(Key.scrapeDmmActressPref, preferenceValue); + } + public Boolean getPromptForUserProvidedURLWhenScraping() { return getBooleanValue(Key.promptForUserProvidedURLWhenScraping, Boolean.FALSE); } @@ -274,4 +283,4 @@ public String getCookieJar() { public void setCookieJar(String preferenceValue) { setStringValue(Key.cookieJar, preferenceValue); } -} +} \ No newline at end of file diff --git a/src/main/java/moviescraper/doctord/scraper/UserAgent.java b/src/main/java/moviescraper/doctord/scraper/UserAgent.java index 1a831f51..a9a105f1 100644 --- a/src/main/java/moviescraper/doctord/scraper/UserAgent.java +++ b/src/main/java/moviescraper/doctord/scraper/UserAgent.java @@ -3,7 +3,8 @@ import java.util.Random; public class UserAgent { - private static final String[] DEFAULT_USER_AGENTS = { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + private static final String[] DEFAULT_USER_AGENTS = { "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/65.0.3325.181 Chrome/65.0.3325.181 Safari/537.36", "Mozilla/5.0 (Linux; Android 7.0; Moto G (5) Build/NPPS25.137-93-8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.137 Mobile Safari/537.36", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_4 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B554a Safari/9537.53", @@ -21,4 +22,4 @@ public static String getUserAgent(int index) { public static String getRandomUserAgent() { return DEFAULT_USER_AGENTS[new Random().nextInt(DEFAULT_USER_AGENTS.length)]; } -} +} \ No newline at end of file diff --git a/src/main/java/moviescraper/doctord/view/GUIMainMenuBar.java b/src/main/java/moviescraper/doctord/view/GUIMainMenuBar.java index e6652281..2c82d492 100644 --- a/src/main/java/moviescraper/doctord/view/GUIMainMenuBar.java +++ b/src/main/java/moviescraper/doctord/view/GUIMainMenuBar.java @@ -39,7 +39,7 @@ public class GUIMainMenuBar extends JMenuBar { /** - * + * */ private static final long serialVersionUID = 1L; @@ -57,7 +57,7 @@ public GUIMainMenuBar(GUIMain guiMain) { /** * Allows you to create a new JCheckBoxMenuItem using Lambda expressions. The preferenceSetterFunction function will be called to change the value when the * menu item is checked and the initial value will be determined by the value returned by preferenceGetterFunction. - * + * * @param checkboxTitle - Text of menu item to create * @param preferenceSetterFunction - setter function called when checkbox item clicked * @param preferenceGetterFunction - function to return initial value of the checkbox @@ -474,6 +474,8 @@ private void initializeScraperConfigurationMenu() { JMenu scraperConfigurationMenu = new JMenu("Scraper's Settings"); scraperConfigurationMenu.setMnemonic(KeyEvent.VK_E); + scraperConfigurationMenu.add(createDmmMenu()); + JMenuItem javlibrary = new JMenuItem("JAVLibrary"); javlibrary.addActionListener(new ActionListener() { @@ -489,6 +491,16 @@ public void actionPerformed(ActionEvent e) { add(scraperConfigurationMenu); } + private JMenu createDmmMenu() { + JMenu submenu = new JMenu("DMM"); + + //Checkbox for option to scrape DMM actress. + JCheckBoxMenuItem scrapeInJapanese = createCheckBoxMenuItem("Scrape Actress", b -> getPreferences().setScrapeDmmActressPref(b), () -> getPreferences().getScrapeDmmActressPref()); + submenu.add(scrapeInJapanese); + + return submenu; + } + private void initializeMenus() { //add the various menus together initializeFileMenu(); @@ -516,4 +528,4 @@ public void enableWriteFile() { } } -} +} \ No newline at end of file