Initial commit
commit
d62969aafc
GPG Key ID:
AD2560A0F84F0759
@ -0,0 +1,6 @@
|
||||
*.sqlite
|
||||
*.sqlite
|
||||
out
|
||||
build
|
||||
.gradle
|
||||
.idea
|
||||
@ -0,0 +1,80 @@
|
||||
plugins {
|
||||
id 'org.jetbrains.kotlin.jvm' version '1.3.72'
|
||||
id 'org.jetbrains.kotlin.kapt' version '1.3.72'
|
||||
id 'application'
|
||||
|
||||
}
|
||||
|
||||
group 'moe.odango'
|
||||
version '1.0-SNAPSHOT'
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
jcenter()
|
||||
maven { url 'https://jitpack.io' }
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
|
||||
|
||||
implementation 'org.xerial:sqlite-jdbc:3.31.1'
|
||||
|
||||
implementation 'io.ktor:ktor-server-core:1.3.2'
|
||||
implementation 'io.ktor:ktor-server-netty:1.3.2'
|
||||
implementation 'io.ktor:ktor-jackson:1.3.2'
|
||||
|
||||
implementation 'io.requery:requery:1.6.1'
|
||||
implementation 'io.requery:requery-kotlin:1.6.1'
|
||||
kapt 'io.requery:requery-processor:1.6.1'
|
||||
|
||||
implementation 'org.slf4j:slf4j-simple:1.6.1'
|
||||
|
||||
implementation 'org.apache.logging.log4j:log4j-core:2.13.3'
|
||||
|
||||
implementation 'com.github.kittinunf.fuel:fuel:2.2.0'
|
||||
implementation 'com.github.kittinunf.fuel:fuel-coroutines:2.2.0'
|
||||
|
||||
implementation 'com.fasterxml:aalto-xml:1.2.1'
|
||||
|
||||
implementation 'org.kodein.di:kodein-di:7.0.0'
|
||||
|
||||
implementation 'com.github.ajalt:clikt:2.7.1'
|
||||
|
||||
implementation 'org.xerial:sqlite-jdbc:3.8.11.2'
|
||||
implementation 'org.postgresql:postgresql:42.2.5'
|
||||
|
||||
implementation 'com.moandjiezana.toml:toml4j:0.7.2'
|
||||
|
||||
implementation 'org.apache.commons:commons-compress:1.18'
|
||||
|
||||
implementation 'com.expediagroup:graphql-kotlin-schema-generator:3.1.1'
|
||||
|
||||
implementation 'com.graphql-java:graphql-java:15.0'
|
||||
|
||||
implementation 'com.github.excitement-engineer:ktor-graphql:1.0.0'
|
||||
|
||||
implementation 'org.jsoup:jsoup:1.13.1'
|
||||
|
||||
implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.7.0'
|
||||
|
||||
implementation 'com.github.jillesvangurp:es-kotlin-wrapper-client:1.0-X-Beta-5-7.7.0'
|
||||
|
||||
testImplementation 'io.kotest:kotest-runner-junit5-jvm:4.0.5'
|
||||
testImplementation 'io.kotest:kotest-assertions-core-jvm:4.0.5'
|
||||
testImplementation 'io.kotest:kotest-property-jvm:4.0.5'
|
||||
}
|
||||
|
||||
application {
|
||||
mainClassName = 'moe.odango.index.MainKt'
|
||||
}
|
||||
|
||||
compileKotlin {
|
||||
kotlinOptions.jvmTarget = '1.8'
|
||||
}
|
||||
compileTestKotlin {
|
||||
kotlinOptions.jvmTarget = '1.8'
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
[database]
|
||||
driver = "sqlite"
|
||||
uri = "jdbc:sqlite:test.sql"
|
||||
|
||||
[elastic]
|
||||
index = "anime"
|
||||
replicas = 0
|
||||
shards = 2
|
||||
@ -0,0 +1,9 @@
|
||||
version: '3.7'
|
||||
services:
|
||||
es:
|
||||
image: elasticsearch:7.7.0
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
ports:
|
||||
- 9200:9200
|
||||
- 9300:9300
|
||||
@ -0,0 +1 @@
|
||||
kotlin.code.style=official
|
||||
Binary file not shown.
@ -0,0 +1,6 @@
|
||||
#Sat Jun 06 16:52:37 CEST 2020
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-6.1-all.zip
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
zipStorePath=wrapper/dists
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
#
|
||||
# Copyright 2015 the original author or authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
##
|
||||
## Gradle start up script for UN*X
|
||||
##
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
# Resolve links: $0 may be a link
|
||||
PRG="$0"
|
||||
# Need this for relative symlinks.
|
||||
while [ -h "$PRG" ] ; do
|
||||
ls=`ls -ld "$PRG"`
|
||||
link=`expr "$ls" : '.*-> \(.*\)$'`
|
||||
if expr "$link" : '/.*' > /dev/null; then
|
||||
PRG="$link"
|
||||
else
|
||||
PRG=`dirname "$PRG"`"/$link"
|
||||
fi
|
||||
done
|
||||
SAVED="`pwd`"
|
||||
cd "`dirname \"$PRG\"`/" >/dev/null
|
||||
APP_HOME="`pwd -P`"
|
||||
cd "$SAVED" >/dev/null
|
||||
|
||||
APP_NAME="Gradle"
|
||||
APP_BASE_NAME=`basename "$0"`
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD="maximum"
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "`uname`" in
|
||||
CYGWIN* )
|
||||
cygwin=true
|
||||
;;
|
||||
Darwin* )
|
||||
darwin=true
|
||||
;;
|
||||
MINGW* )
|
||||
msys=true
|
||||
;;
|
||||
NONSTOP* )
|
||||
nonstop=true
|
||||
;;
|
||||
esac
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD="java"
|
||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
||||
MAX_FD_LIMIT=`ulimit -H -n`
|
||||
if [ $? -eq 0 ] ; then
|
||||
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
||||
MAX_FD="$MAX_FD_LIMIT"
|
||||
fi
|
||||
ulimit -n $MAX_FD
|
||||
if [ $? -ne 0 ] ; then
|
||||
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
||||
fi
|
||||
else
|
||||
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# For Darwin, add options to specify how the application appears in the dock
|
||||
if $darwin; then
|
||||
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
||||
fi
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
|
||||
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
||||
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
||||
JAVACMD=`cygpath --unix "$JAVACMD"`
|
||||
|
||||
# We build the pattern for arguments to be converted via cygpath
|
||||
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
||||
SEP=""
|
||||
for dir in $ROOTDIRSRAW ; do
|
||||
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
||||
SEP="|"
|
||||
done
|
||||
OURCYGPATTERN="(^($ROOTDIRS))"
|
||||
# Add a user-defined pattern to the cygpath arguments
|
||||
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
||||
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
||||
fi
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
i=0
|
||||
for arg in "$@" ; do
|
||||
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
||||
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
||||
|
||||
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
||||
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
||||
else
|
||||
eval `echo args$i`="\"$arg\""
|
||||
fi
|
||||
i=`expr $i + 1`
|
||||
done
|
||||
case $i in
|
||||
0) set -- ;;
|
||||
1) set -- "$args0" ;;
|
||||
2) set -- "$args0" "$args1" ;;
|
||||
3) set -- "$args0" "$args1" "$args2" ;;
|
||||
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
||||
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
||||
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
||||
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
||||
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
||||
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Escape application args
|
||||
save () {
|
||||
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
||||
echo " "
|
||||
}
|
||||
APP_ARGS=`save "$@"`
|
||||
|
||||
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
||||
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
@ -0,0 +1,100 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%" == "" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%" == "" set DIRNAME=.
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if "%ERRORLEVEL%" == "0" goto init
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto init
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:init
|
||||
@rem Get command-line arguments, handling Windows variants
|
||||
|
||||
if not "%OS%" == "Windows_NT" goto win9xME_args
|
||||
|
||||
:win9xME_args
|
||||
@rem Slurp the command line arguments.
|
||||
set CMD_LINE_ARGS=
|
||||
set _SKIP=2
|
||||
|
||||
:win9xME_args_slurp
|
||||
if "x%~1" == "x" goto execute
|
||||
|
||||
set CMD_LINE_ARGS=%*
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if "%ERRORLEVEL%"=="0" goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
||||
exit /b 1
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
||||
@ -0,0 +1,2 @@
|
||||
rootProject.name = 'index'
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
package moe.odango.index.cli
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import com.github.ajalt.clikt.parameters.options.option
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import moe.odango.index.sync.AniDBTitleSync
|
||||
import java.io.File
|
||||
|
||||
class AniDBSync : CliktCommand(name = "anidb",help = "Sync the titles from AniDB with the local database") {
|
||||
private val file by option(help = "Read from file instead of downloading")
|
||||
|
||||
override fun run() {
|
||||
val sync = AniDBTitleSync()
|
||||
val file = file
|
||||
|
||||
runBlocking {
|
||||
if (file == null) {
|
||||
sync.run()
|
||||
} else {
|
||||
val fileObj = File(file).takeIf { it.isFile } ?: throw RuntimeException("File doesn't exist")
|
||||
sync.syncWithFile(fileObj)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package moe.odango.index.cli
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import com.github.ajalt.clikt.parameters.options.flag
|
||||
import com.github.ajalt.clikt.parameters.options.option
|
||||
import io.requery.sql.KotlinConfiguration
|
||||
import io.requery.sql.SchemaModifier
|
||||
import io.requery.sql.TableCreationMode
|
||||
import moe.odango.index.di
|
||||
import org.kodein.di.instance
|
||||
|
||||
class DatabaseMigration : CliktCommand(name = "db:create") {
|
||||
private val print by option().flag(default = false)
|
||||
private val exec by option().flag(default = false)
|
||||
|
||||
override fun run() {
|
||||
val config by di.instance<KotlinConfiguration>()
|
||||
val modifier = SchemaModifier(config)
|
||||
|
||||
if (exec) {
|
||||
modifier.createTables(TableCreationMode.CREATE_NOT_EXISTS)
|
||||
}
|
||||
|
||||
if (print) {
|
||||
println(modifier.createTablesString(TableCreationMode.CREATE_NOT_EXISTS))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package moe.odango.index.cli
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import moe.odango.index.es.Indexer
|
||||
|
||||
class ElasticIndex : CliktCommand(name = "elastic:index") {
|
||||
override fun run() {
|
||||
Indexer().run()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package moe.odango.index.cli
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import moe.odango.index.http.Server
|
||||
|
||||
class HTTPServer : CliktCommand(name = "http:serve") {
|
||||
override fun run() {
|
||||
Server().run()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package moe.odango.index.cli
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import moe.odango.index.sync.MyAnimeListListingSync
|
||||
|
||||
class MyAnimeListListingSync : CliktCommand(name = "mal:sync-listing") {
|
||||
override fun run() {
|
||||
runBlocking {
|
||||
MyAnimeListListingSync().run()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package moe.odango.index.cli
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import moe.odango.index.sync.MyAnimeListPageSync
|
||||
|
||||
class MyAnimeListPageSync : CliktCommand(name = "mal:sync-page") {
|
||||
override fun run() {
|
||||
runBlocking {
|
||||
MyAnimeListPageSync().run()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
package moe.odango.index.config
|
||||
|
||||
data class DatabaseConfiguration(
|
||||
val uri: String,
|
||||
val driver: String = "sqlite",
|
||||
val username: String? = null,
|
||||
val password: String? = null
|
||||
)
|
||||
@ -0,0 +1,8 @@
|
||||
package moe.odango.index.config
|
||||
|
||||
data class ElasticSearchConfiguration(
|
||||
val host: String = "localhost:9200",
|
||||
val index: String = "index_anime",
|
||||
val replicas: Int = 0,
|
||||
val shards: Int = 2
|
||||
)
|
||||
@ -0,0 +1,6 @@
|
||||
package moe.odango.index.config
|
||||
|
||||
data class IndexConfiguration(
|
||||
val database: DatabaseConfiguration = DatabaseConfiguration("jdbc:sqlite:index.sqlite"),
|
||||
val elastic: ElasticSearchConfiguration = ElasticSearchConfiguration()
|
||||
)
|
||||
@ -0,0 +1,59 @@
|
||||
package moe.odango.index
|
||||
|
||||
import com.moandjiezana.toml.Toml
|
||||
import io.inbot.eskotlinwrapper.IndexRepository
|
||||
import io.requery.Persistable
|
||||
import io.requery.sql.KotlinConfiguration
|
||||
import io.requery.sql.KotlinEntityDataStore
|
||||
import moe.odango.index.config.IndexConfiguration
|
||||
import moe.odango.index.entity.Models
|
||||
import moe.odango.index.es.dto.AnimeDTO
|
||||
import org.apache.http.HttpHost
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.client.indexRepository
|
||||
import org.kodein.di.*
|
||||
import org.sqlite.SQLiteDataSource
|
||||
import java.io.File
|
||||
import javax.sql.CommonDataSource
|
||||
|
||||
val di = DI {
|
||||
bind<RestHighLevelClient>() with provider {
|
||||
RestHighLevelClient(RestClient.builder(HttpHost.create(instance<IndexConfiguration>().elastic.host)))
|
||||
}
|
||||
|
||||
bind<IndexRepository<AnimeDTO>>() with provider {
|
||||
instance<RestHighLevelClient>().indexRepository<AnimeDTO>(instance<IndexConfiguration>().elastic.index)
|
||||
}
|
||||
|
||||
bind<KotlinConfiguration>() with singleton { KotlinConfiguration(Models.DEFAULT, dataSource = instance()) }
|
||||
bind<KotlinEntityDataStore<Persistable>>() with singleton { KotlinEntityDataStore<Persistable>(instance()) }
|
||||
constant(tag = "config-file") with mutableListOf(
|
||||
"/etc/index/config.toml",
|
||||
System.getenv("HOME") + "/.index/config.toml"
|
||||
)
|
||||
|
||||
bind<IndexConfiguration>() with singleton {
|
||||
val toml = instance<MutableList<String>>("config-file")
|
||||
.find { File(it).isFile }
|
||||
?.let { File(it).readText(Charsets.UTF_8) }
|
||||
|
||||
toml?.let {
|
||||
Toml()
|
||||
.read(toml)
|
||||
.to(IndexConfiguration::class.java)
|
||||
} ?: IndexConfiguration()
|
||||
}
|
||||
|
||||
bind<CommonDataSource>() with singleton {
|
||||
val config: IndexConfiguration = instance()
|
||||
|
||||
when (config.database.driver) {
|
||||
"sqlite" -> SQLiteDataSource().apply {
|
||||
url = config.database.uri
|
||||
}
|
||||
|
||||
else -> throw RuntimeException("Database driver '${config.database.driver}' not recognized")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface AniDBInfo : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:OneToOne
|
||||
@get:ForeignKey
|
||||
val anime: Anime
|
||||
|
||||
var type: ReleaseType
|
||||
var restricted: Boolean
|
||||
var episodes: Int?
|
||||
var description: String
|
||||
var image: String?
|
||||
var startYear: Int?
|
||||
var startMonth: Int?
|
||||
var startDay: Int?
|
||||
|
||||
var endYear: Int?
|
||||
var endMonth: Int?
|
||||
var endDay: Int?
|
||||
|
||||
enum class ReleaseType {
|
||||
Movie,
|
||||
MusicVideo,
|
||||
OVA,
|
||||
Other,
|
||||
TVSeries,
|
||||
TVSpecial,
|
||||
Web,
|
||||
Unknown;
|
||||
|
||||
companion object {
|
||||
fun fromAniDBString(value: String): ReleaseType = when (value.toLowerCase()) {
|
||||
"movie" -> Movie
|
||||
"music video" -> MusicVideo
|
||||
"ova" -> OVA
|
||||
"other" -> Other
|
||||
"tv series" -> TVSeries
|
||||
"tv special" -> TVSpecial
|
||||
"web" -> Web
|
||||
else -> Unknown
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
companion object : EntityHelper<AniDBInfoEntity> by helper(::AniDBInfoEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface Anime : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:Index
|
||||
var aniDbId: Long?
|
||||
|
||||
@get:OneToOne
|
||||
val aniDbInfo: AniDBInfo?
|
||||
|
||||
@get:OneToMany(mappedBy = "anime")
|
||||
val titles: List<Title>
|
||||
|
||||
@get:Index
|
||||
var myAnimeListId: Long?
|
||||
|
||||
@get:ManyToMany
|
||||
@get:JunctionTable(type = AnimeProducer::class)
|
||||
val producers: List<Producer>
|
||||
|
||||
@get:OneToOne
|
||||
val myAnimeListInfo: MyAnimeListInfo?
|
||||
|
||||
@get:OneToMany(mappedBy = "anime_from")
|
||||
val relatedTo: List<AnimeRelation>
|
||||
|
||||
@get:OneToMany(mappedBy = "anime_to")
|
||||
val relatedFrom: List<AnimeRelation>
|
||||
|
||||
@get:OneToMany(mappedBy = "anime")
|
||||
val genres: List<AnimeGenre>
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
var series: AnimeSeries?
|
||||
|
||||
val replacedWith: UUID?
|
||||
|
||||
companion object : EntityHelper<AnimeEntity> by helper(::AnimeEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface AnimeGenre : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
val anime: Anime
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
val genre: Genre
|
||||
|
||||
val source: InfoSource
|
||||
|
||||
companion object : EntityHelper<AnimeGenreEntity> by helper(::AnimeGenreEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.ProducerFunction
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface AnimeProducer : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:ManyToOne
|
||||
val anime: Anime
|
||||
|
||||
@get:ManyToOne
|
||||
val producer: Producer
|
||||
|
||||
@get:Column(name = "producer_function")
|
||||
var function: ProducerFunction
|
||||
val source: InfoSource
|
||||
|
||||
companion object : EntityHelper<AnimeProducerEntity> by helper(::AnimeProducerEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,82 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface AnimeRelation : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
@get:Column(name = "anime_from")
|
||||
val from: Anime
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
@get:Column(name = "anime_to")
|
||||
val to: Anime
|
||||
|
||||
var relation: RelationType
|
||||
val source: InfoSource
|
||||
|
||||
companion object : EntityHelper<AnimeRelationEntity> by helper(::AnimeRelationEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
|
||||
enum class RelationType {
|
||||
Prequel,
|
||||
Sequel,
|
||||
SideStory,
|
||||
FullStory,
|
||||
ParentStory,
|
||||
Summary,
|
||||
SpinOff,
|
||||
AlternateVersion,
|
||||
AlternateSetting,
|
||||
SameSetting,
|
||||
Other,
|
||||
Character;
|
||||
|
||||
val inverse: RelationType
|
||||
get() {
|
||||
return when (this) {
|
||||
Prequel -> Sequel
|
||||
Sequel -> Prequel
|
||||
Summary -> FullStory
|
||||
FullStory -> Summary
|
||||
SideStory -> ParentStory
|
||||
SpinOff -> ParentStory
|
||||
ParentStory -> SideStory
|
||||
AlternateSetting -> AlternateSetting
|
||||
AlternateVersion -> AlternateVersion
|
||||
SameSetting -> SameSetting
|
||||
Other -> Other
|
||||
Character -> Character
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun fromString(type: String): RelationType {
|
||||
return when (type.toLowerCase()) {
|
||||
"alternate setting" -> AlternateSetting
|
||||
"alternate version" -> AlternateVersion
|
||||
"prequel" -> Prequel
|
||||
"sequel" -> Sequel
|
||||
"spin off" -> SpinOff
|
||||
"summary" -> Summary
|
||||
"side story" -> SideStory
|
||||
"full story" -> FullStory
|
||||
"parent story" -> ParentStory
|
||||
"character" -> Character
|
||||
"same setting" -> SameSetting
|
||||
else -> Other
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.Entity
|
||||
import io.requery.Key
|
||||
import io.requery.Persistable
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface AnimeSeries : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
val name: String?
|
||||
var replacedWith: UUID?
|
||||
|
||||
companion object : EntityHelper<AnimeSeriesEntity> by helper(::AnimeSeriesEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface AnimeTag : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
val anime: Anime
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
val tag: Tag
|
||||
val source: InfoSource
|
||||
val spoiler: Boolean
|
||||
|
||||
companion object : EntityHelper<AnimeTagEntity> by helper(::AnimeTagEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.Entity
|
||||
import io.requery.Key
|
||||
import io.requery.Persistable
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface Genre : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
val myAnimeListId: Int
|
||||
val name: String
|
||||
|
||||
companion object : EntityHelper<GenreEntity> by helper(::GenreEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.scraper.mal.AnimePageScraper
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.ISOTextConverter
|
||||
import moe.odango.index.utils.helper
|
||||
import moe.odango.index.utils.toIntDate
|
||||
import org.joda.time.DateTime
|
||||
import java.sql.Date
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface MyAnimeListInfo : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:OneToOne
|
||||
@get:ForeignKey
|
||||
val anime: Anime
|
||||
|
||||
var releaseType: ReleaseType
|
||||
var episodes: Int?
|
||||
|
||||
@get:Convert(ISOTextConverter::class)
|
||||
var lastScrape: DateTime?
|
||||
var rating: Rating?
|
||||
var image: String?
|
||||
var description: String?
|
||||
var source: String?
|
||||
var airedStart: Date?
|
||||
var airedEnd: Date?
|
||||
var premieredSeason: String?
|
||||
var premieredYear: Int?
|
||||
var duration: Int?
|
||||
|
||||
|
||||
@get:Transient
|
||||
val aired: AnimePageScraper.Aired?
|
||||
get() = airedStart?.let { AnimePageScraper.Aired(it.toIntDate(), airedEnd?.toIntDate()) }
|
||||
|
||||
@get:Transient
|
||||
val premiered: AnimePageScraper.Premiered?
|
||||
get() = premieredSeason
|
||||
?.let(AnimePageScraper.Premiered.Season.Companion::fromString)
|
||||
?.let { s ->
|
||||
premieredYear?.let { y ->
|
||||
AnimePageScraper.Premiered(s, y)
|
||||
}
|
||||
}
|
||||
|
||||
enum class Rating {
|
||||
G,
|
||||
PG,
|
||||
PG13,
|
||||
R,
|
||||
RPlus,
|
||||
Rx;
|
||||
|
||||
companion object {
|
||||
fun fromString(rating: String): Rating? = when (rating) {
|
||||
"G" -> G
|
||||
"PG" -> PG
|
||||
"PG-13" -> PG13
|
||||
"R" -> R
|
||||
"R+" -> RPlus
|
||||
"Rx" -> Rx
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum class ReleaseType {
|
||||
TV,
|
||||
ONA,
|
||||
OVA,
|
||||
Movie,
|
||||
Special,
|
||||
Music,
|
||||
Unknown
|
||||
}
|
||||
|
||||
companion object : EntityHelper<MyAnimeListInfoEntity> by helper(::MyAnimeListInfoEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
episodes = null
|
||||
releaseType = ReleaseType.Unknown
|
||||
lastScrape = null
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface Producer : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
val myAnimeListId: Int?
|
||||
val name: String
|
||||
|
||||
@get:ManyToMany
|
||||
val animes: List<Anime>
|
||||
|
||||
companion object : EntityHelper<ProducerEntity> by helper(::ProducerEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.Entity
|
||||
import io.requery.Key
|
||||
import io.requery.Persistable
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface Tag : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
val aniDbId: Long?
|
||||
|
||||
val parentId: UUID
|
||||
val name: String
|
||||
val description: String
|
||||
val spoiler: Boolean
|
||||
|
||||
companion object : EntityHelper<TagEntity> by helper(::TagEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
package moe.odango.index.entity
|
||||
|
||||
import io.requery.*
|
||||
import moe.odango.index.utils.EntityHelper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.helper
|
||||
import java.util.*
|
||||
|
||||
@Entity
|
||||
interface Title : Persistable {
|
||||
@get:Key
|
||||
val id: UUID
|
||||
|
||||
@get:Index
|
||||
var name: String
|
||||
|
||||
@get:ManyToOne
|
||||
@get:ForeignKey
|
||||
val anime: Anime
|
||||
|
||||
val source: InfoSource
|
||||
|
||||
|
||||
@get:Column(name = "lang")
|
||||
val language: String?
|
||||
val type: TitleType
|
||||
var hidden: Boolean
|
||||
|
||||
|
||||
enum class TitleType {
|
||||
Synonym,
|
||||
TitleCard,
|
||||
Kana,
|
||||
Short,
|
||||
Official,
|
||||
Main;
|
||||
|
||||
fun toAniDBString() = when (this) {
|
||||
Main -> "main"
|
||||
Synonym -> "syn"
|
||||
TitleCard -> "card"
|
||||
Kana -> "kana"
|
||||
Short -> "short"
|
||||
Official -> "official"
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun fromAniDBString(type: String) = when (type) {
|
||||
"card" -> TitleCard
|
||||
"official" -> Official
|
||||
"short" -> Short
|
||||
"syn" -> Synonym
|
||||
"kana" -> Kana
|
||||
"main" -> Main
|
||||
else -> Synonym
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object : EntityHelper<TitleEntity> by helper(::TitleEntity, {
|
||||
setId(UUID.randomUUID())
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,141 @@
|
||||
package moe.odango.index.es
|
||||
|
||||
import io.inbot.eskotlinwrapper.IndexRepository
|
||||
import io.requery.Persistable
|
||||
import io.requery.kotlin.invoke
|
||||
import io.requery.sql.KotlinEntityDataStore
|
||||
import moe.odango.index.config.IndexConfiguration
|
||||
import moe.odango.index.di
|
||||
import moe.odango.index.entity.Anime
|
||||
import moe.odango.index.es.dto.AnimeDescriptionDTO
|
||||
import moe.odango.index.es.dto.AnimeTitleDTO
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.client.configure
|
||||
import org.elasticsearch.client.indices.GetIndexRequest
|
||||
import org.kodein.di.instance
|
||||
import moe.odango.index.es.dto.AnimeDTO as AnimeDTO
|
||||
|
||||
class Indexer {
|
||||
private val indexRepo by di.instance<IndexRepository<AnimeDTO>>()
|
||||
private val indexConfig by di.instance<IndexConfiguration>()
|
||||
private val client by di.instance<RestHighLevelClient>()
|
||||
private val config by lazy { indexConfig.elastic }
|
||||
private val entityStore by di.instance<KotlinEntityDataStore<Persistable>>()
|
||||
|
||||
fun run() {
|
||||
createIndex()
|
||||
index()
|
||||
client.close()
|
||||
}
|
||||
|
||||
fun createIndex() {
|
||||
val indexExists = client.indices()
|
||||
.exists(GetIndexRequest(config.index), RequestOptions.DEFAULT);
|
||||
|
||||
if (indexExists)
|
||||
return
|
||||
|
||||
indexRepo.createIndex {
|
||||
configure {
|
||||
settings {
|
||||
replicas = config.replicas
|
||||
shards = config.shards
|
||||
|
||||
addTokenizer("autocomplete") {
|
||||
this["type"] = "edge_ngram"
|
||||
this["min_gram"] = 2
|
||||
this["max_gram"] = 10
|
||||
this["token_chars"] = listOf("letter")
|
||||
}
|
||||
|
||||
addAnalyzer("autocomplete") {
|
||||
this["tokenizer"] = "autocomplete"
|
||||
this["filter"] = listOf("lowercase")
|
||||
}
|
||||
|
||||
addAnalyzer("autocomplete_search") {
|
||||
this["tokenizer"] = "lowercase"
|
||||
}
|
||||
}
|
||||
|
||||
mappings {
|
||||
nestedField("title") {
|
||||
text("name") {
|
||||
analyzer = "autocomplete"
|
||||
searchAnalyzer = "autocomplete_search"
|
||||
}
|
||||
}
|
||||
|
||||
nestedField("description") {
|
||||
text("text") {
|
||||
analyzer = "standard"
|
||||
searchAnalyzer = "standard"
|
||||
}
|
||||
}
|
||||
|
||||
keyword("genre")
|
||||
|
||||
objField("premiered") {
|
||||
keyword("season")
|
||||
number<Int>("year")
|
||||
}
|
||||
|
||||
objField("aired") {
|
||||
field("start", "date")
|
||||
field("end", "date")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun index() {
|
||||
var i = 0;
|
||||
entityStore {
|
||||
val q = select(Anime::class)
|
||||
|
||||
indexRepo.bulk(50) {
|
||||
for (item in q()) {
|
||||
if (item.replacedWith != null) continue;
|
||||
i++
|
||||
if (i % 1_000 == 0) {
|
||||
println(" => $i - ${item.id}")
|
||||
}
|
||||
|
||||
index(
|
||||
item.id.toString(),
|
||||
AnimeDTO(
|
||||
item.id,
|
||||
item.titles.map {
|
||||
AnimeTitleDTO(
|
||||
it.name,
|
||||
it.language,
|
||||
it.type.toString(),
|
||||
it.source
|
||||
)
|
||||
},
|
||||
item.myAnimeListInfo?.let {
|
||||
it.description?.let { desc ->
|
||||
listOf(
|
||||
AnimeDescriptionDTO(
|
||||
desc,
|
||||
InfoSource.MyAnimeList
|
||||
)
|
||||
)
|
||||
}
|
||||
} ?: listOf(),
|
||||
item.genres.map { it.genre.name }.toSet().toList(),
|
||||
item.myAnimeListInfo?.premiered,
|
||||
item.myAnimeListInfo?.aired
|
||||
),
|
||||
false
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println(" => Indexed $i entries.")
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package moe.odango.index.es.dto
|
||||
|
||||
import moe.odango.index.scraper.mal.AnimePageScraper
|
||||
import java.util.*
|
||||
|
||||
data class AnimeDTO(
|
||||
val id: UUID,
|
||||
val title: Collection<AnimeTitleDTO>,
|
||||
val description: Collection<AnimeDescriptionDTO>,
|
||||
val genre: Collection<String>,
|
||||
val premiered: AnimePageScraper.Premiered?,
|
||||
val aired: AnimePageScraper.Aired?
|
||||
)
|
||||
@ -0,0 +1,8 @@
|
||||
package moe.odango.index.es.dto
|
||||
|
||||
import moe.odango.index.utils.InfoSource
|
||||
|
||||
data class AnimeDescriptionDTO(
|
||||
val text: String,
|
||||
val source: InfoSource
|
||||
)
|
||||
@ -0,0 +1,10 @@
|
||||
package moe.odango.index.es.dto
|
||||
|
||||
import moe.odango.index.utils.InfoSource
|
||||
|
||||
data class AnimeTitleDTO(
|
||||
val name: String,
|
||||
val language: String?,
|
||||
val type: String,
|
||||
val source: InfoSource
|
||||
)
|
||||
@ -0,0 +1,41 @@
|
||||
package moe.odango.index.http
|
||||
|
||||
import com.expediagroup.graphql.SchemaGeneratorConfig
|
||||
import com.expediagroup.graphql.TopLevelObject
|
||||
import com.expediagroup.graphql.toSchema
|
||||
import io.ktor.http.content.resource
|
||||
import io.ktor.http.content.resources
|
||||
import io.ktor.http.content.static
|
||||
import io.ktor.routing.routing
|
||||
import io.ktor.server.engine.embeddedServer
|
||||
import io.ktor.server.netty.Netty
|
||||
import ktor.graphql.GraphQLRouteConfig
|
||||
import ktor.graphql.graphQL
|
||||
import moe.odango.index.http.graphql.AnimeService
|
||||
|
||||
class Server {
|
||||
fun run() {
|
||||
embeddedServer(Netty, port = 3336) {
|
||||
routing {
|
||||
|
||||
graphQL("/graphql",toSchema(
|
||||
SchemaGeneratorConfig(listOf("moe.odango.index")),
|
||||
listOf(TopLevelObject(AnimeService()))
|
||||
)) {
|
||||
GraphQLRouteConfig(
|
||||
graphiql = true
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
static {
|
||||
resource("/", "web/index.html")
|
||||
static("/") {
|
||||
resources("web")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.start(true)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,100 @@
|
||||
package moe.odango.index.http.graphql
|
||||
|
||||
import io.inbot.eskotlinwrapper.IndexRepository
|
||||
import io.inbot.eskotlinwrapper.dsl.*
|
||||
import moe.odango.index.di
|
||||
import moe.odango.index.entity.Title
|
||||
import moe.odango.index.es.dto.AnimeDTO
|
||||
import moe.odango.index.http.graphql.dto.AnimeItem
|
||||
import moe.odango.index.http.graphql.dto.AnimeTitleItem
|
||||
import moe.odango.index.http.graphql.dto.AutoCompleteItem
|
||||
import moe.odango.index.utils.ScoreMode
|
||||
import moe.odango.index.utils.nested
|
||||
import org.elasticsearch.action.search.dsl
|
||||
import org.elasticsearch.action.search.source
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.Fuzziness
|
||||
import org.elasticsearch.search.suggest.SuggestBuilder
|
||||
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder
|
||||
import org.kodein.di.instance
|
||||
|
||||
class AnimeService {
|
||||
private val es by di.instance<IndexRepository<AnimeDTO>>()
|
||||
private val client by di.instance<RestHighLevelClient>()
|
||||
|
||||
fun autocomplete(title: String): List<AutoCompleteItem> {
|
||||
return es.search {
|
||||
source {
|
||||
suggest(
|
||||
SuggestBuilder().addSuggestion(
|
||||
"title",
|
||||
CompletionSuggestionBuilder("title.name").prefix(title, Fuzziness.AUTO)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
.hits
|
||||
.mapNotNull { (s, item) ->
|
||||
AutoCompleteItem(
|
||||
s.innerHits["title"]?.hits?.get(0)?.sourceAsMap?.get("name")?.toString() ?: return@mapNotNull null,
|
||||
item?.id.toString() ?: return@mapNotNull null
|
||||
)
|
||||
}
|
||||
.toList()
|
||||
}
|
||||
|
||||
fun search(
|
||||
title: String? = null,
|
||||
description: String? = null,
|
||||
genre: List<String>? = null
|
||||
): List<AnimeItem> {
|
||||
val items = mutableListOf<ESQuery>()
|
||||
|
||||
title?.let {
|
||||
items.add(
|
||||
nested("title") {
|
||||
query = MatchQuery("title.name", it)
|
||||
scoreMode = ScoreMode.max
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
genre?.let {
|
||||
items.add(
|
||||
bool {
|
||||
must(*it.toSet().map { genre -> TermQuery("genre", genre) }.toTypedArray())
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
description?.let {
|
||||
items.add(
|
||||
nested("description") {
|
||||
query = MatchQuery("description.name", it)
|
||||
scoreMode = ScoreMode.max
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
return es.search {
|
||||
dsl {
|
||||
query = disMax {
|
||||
queries(*items.toTypedArray())
|
||||
}
|
||||
}
|
||||
}
|
||||
.hits
|
||||
.mapNotNull { (_, dto) ->
|
||||
dto?.let {
|
||||
AnimeItem(
|
||||
it.id.toString(),
|
||||
it.title.map { dto ->
|
||||
AnimeTitleItem(dto.name, dto.language, dto.type.let(Title.TitleType::valueOf), dto.source)
|
||||
},
|
||||
it.genre.toSet().toList()
|
||||
)
|
||||
}
|
||||
}
|
||||
.toList()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package moe.odango.index.http.graphql.dto
|
||||
|
||||
data class AnimeItem(
|
||||
val id: String,
|
||||
val titles: List<AnimeTitleItem>,
|
||||
val genres: List<String>
|
||||
)
|
||||
@ -0,0 +1,11 @@
|
||||
package moe.odango.index.http.graphql.dto
|
||||
|
||||
import moe.odango.index.entity.Title
|
||||
import moe.odango.index.utils.InfoSource
|
||||
|
||||
data class AnimeTitleItem(
|
||||
val name: String,
|
||||
val language: String?,
|
||||
val type: Title.TitleType,
|
||||
val source: InfoSource
|
||||
)
|
||||
@ -0,0 +1,6 @@
|
||||
package moe.odango.index.http.graphql.dto
|
||||
|
||||
data class AutoCompleteItem(
|
||||
val title: String,
|
||||
val id: String
|
||||
)
|
||||
@ -0,0 +1,30 @@
|
||||
package moe.odango.index
|
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import com.github.ajalt.clikt.core.subcommands
|
||||
import com.github.ajalt.clikt.parameters.options.option
|
||||
import moe.odango.index.cli.*
|
||||
import org.kodein.di.instance
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
object : CliktCommand() {
|
||||
val config: String? by option()
|
||||
|
||||
init {
|
||||
subcommands(
|
||||
AniDBSync(),
|
||||
DatabaseMigration(),
|
||||
MyAnimeListListingSync(),
|
||||
MyAnimeListPageSync(),
|
||||
HTTPServer(),
|
||||
ElasticIndex()
|
||||
)
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
val configFile by di.instance<MutableList<String>>("config-file")
|
||||
config?.let { configFile.add(0, it) }
|
||||
}
|
||||
}
|
||||
.main(args)
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
package moe.odango.index.scraper.mal
|
||||
|
||||
import moe.odango.index.entity.MyAnimeListInfo
|
||||
import org.jsoup.Jsoup
|
||||
import java.net.URI
|
||||
|
||||
class AnimeListScraper(body: String) {
|
||||
private val dom = Jsoup.parse(body)
|
||||
|
||||
data class MyAnimeListListingItem(val myAnimeListId: Long, val title: String, val type: MyAnimeListInfo.ReleaseType, val episodes: Int?)
|
||||
|
||||
fun getItems(): List<MyAnimeListListingItem> {
|
||||
val arr = dom
|
||||
.select("[id=content] .list tbody tr")
|
||||
.toList()
|
||||
|
||||
return arr.drop(1).mapNotNull {
|
||||
val link = it.select("a[id^=sinfo]").first()
|
||||
val id = link
|
||||
.attr("href")
|
||||
.split("/")
|
||||
.let {
|
||||
it[it.indexOf("anime") + 1].toLongOrNull()
|
||||
}
|
||||
|
||||
val title = link.text()
|
||||
val type = it.child(2).text()
|
||||
val eps = it.child(3).text().toIntOrNull()
|
||||
|
||||
id?.let { MyAnimeListListingItem(id, title, MyAnimeListInfo.ReleaseType.valueOf(type), eps) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,246 @@
|
||||
package moe.odango.index.scraper.mal
|
||||
|
||||
import moe.odango.index.entity.AnimeRelation
|
||||
import moe.odango.index.entity.MyAnimeListInfo
|
||||
import moe.odango.index.utils.IntDate
|
||||
import moe.odango.index.utils.ProducerFunction
|
||||
import moe.odango.index.utils.brText
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Element
|
||||
import java.time.Duration
|
||||
import java.util.*
|
||||
|
||||
class AnimePageScraper(body: String) {
|
||||
private val dom = Jsoup.parse(body)
|
||||
|
||||
private val items: MutableMap<String, Pair<String, Element>> = mutableMapOf()
|
||||
|
||||
data class Info(
|
||||
val title: String,
|
||||
val englishName: String?,
|
||||
val japaneseName: String?,
|
||||
val synonyms: List<String>,
|
||||
val description: String,
|
||||
val type: MyAnimeListInfo.ReleaseType,
|
||||
val episodes: Int?,
|
||||
val source: String?,
|
||||
val image: String?,
|
||||
val genres: List<Genre>,
|
||||
val aired: Aired?,
|
||||
val premiered: Premiered?,
|
||||
val rating: MyAnimeListInfo.Rating?,
|
||||
val duration: Duration?,
|
||||
val related: List<Relation>,
|
||||
val producers: List<ProducerRelation>
|
||||
)
|
||||
|
||||
fun getInfo() = Info(
|
||||
getTitle(),
|
||||
getEnglishName(),
|
||||
getJapaneseName(),
|
||||
getSynonyms(),
|
||||
getDescription(),
|
||||
getReleaseType(),
|
||||
getEpisodes(),
|
||||
getSource(),
|
||||
getImage(),
|
||||
getGenres(),
|
||||
getAired(),
|
||||
getPremiered(),
|
||||
getRating(),
|
||||
getDuration(),
|
||||
getRelated(),
|
||||
getProducers()
|
||||
)
|
||||
|
||||
fun getEpisodes(): Int? {
|
||||
return items["episodes"]?.first?.toIntOrNull()
|
||||
}
|
||||
|
||||
fun getReleaseType(): MyAnimeListInfo.ReleaseType {
|
||||
return MyAnimeListInfo.ReleaseType.valueOf(items["type"]!!.first)
|
||||
}
|
||||
|
||||
fun getTitle(): String {
|
||||
return dom.select(".h1-title span[itemprop=name]").first().brText().split("\n").first()
|
||||
}
|
||||
|
||||
fun getSynonyms(): List<String> {
|
||||
// If the english name or title contains a comma we can't parse synonyms since they're splice by comma's
|
||||
return if (getEnglishName()?.contains(",") == true || getTitle().contains(",")) listOf() else items["synonyms"]?.first?.split(
|
||||
","
|
||||
)?.map { it.trim() } ?: listOf()
|
||||
}
|
||||
|
||||
fun getJapaneseName(): String? {
|
||||
return items["japanese"]?.first
|
||||
}
|
||||
|
||||
fun getEnglishName(): String? {
|
||||
return items["english"]?.first
|
||||
}
|
||||
|
||||
init {
|
||||
val after = dom
|
||||
.select(".dark_text")
|
||||
.parents()
|
||||
|
||||
for (item in after) {
|
||||
val parts = item.text().split(":", limit = 2)
|
||||
val key = parts.first().trim().toLowerCase()
|
||||
val value = parts.last().trim()
|
||||
items[key] = value to item
|
||||
}
|
||||
}
|
||||
|
||||
fun getSource(): String? {
|
||||
return items["source"]?.first
|
||||
}
|
||||
|
||||
fun getImage(): String? {
|
||||
return dom.selectFirst("img[itemprop=image]")?.attr("data-src")
|
||||
}
|
||||
|
||||
fun getDescription(): String {
|
||||
return dom.selectFirst("[itemprop=description]")?.brText() ?: ""
|
||||
}
|
||||
|
||||
fun getAired(): Aired? {
|
||||
return items["aired"]?.let {
|
||||
val (from, to) = it.first.split(" to ", limit = 2)
|
||||
.let { part -> part.first() to part.getOrNull(1) }
|
||||
|
||||
return Aired(IntDate.parse("MMM d, yyyy", from, Locale.US) ?: return null, to?.let { dateStr ->
|
||||
IntDate.parse("MMM d, yyyy", dateStr, Locale.US)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
data class Aired(val start: IntDate, val end: IntDate? = null)
|
||||
|
||||
fun getPremiered(): Premiered? {
|
||||
val (season, year) = items["premiered"]
|
||||
?.first
|
||||
?.split(" ", limit = 2)
|
||||
?.takeUnless { it.size != 2 }
|
||||
?: return null
|
||||
|
||||
return Premiered(Premiered.Season.fromString(season) ?: return null, year.toIntOrNull() ?: return null)
|
||||
}
|
||||
|
||||
data class Premiered(val season: Season, val year: Int) {
|
||||
enum class Season {
|
||||
Spring,
|
||||
Summer,
|
||||
Fall,
|
||||
Winter;
|
||||
|
||||
companion object {
|
||||
fun fromString(season: String): Season? {
|
||||
return when (season.toLowerCase()) {
|
||||
"spring" -> Spring
|
||||
"summer" -> Summer
|
||||
"fall" -> Fall
|
||||
"winter" -> Winter
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getGenres(): List<Genre> {
|
||||
return items["genres"]?.second?.let {
|
||||
it.select("a").map { a ->
|
||||
val href = a.attr("href").split("/")
|
||||
Genre(href[href.indexOf("genre") + 1].toInt(), a.text().trim())
|
||||
}
|
||||
} ?: emptyList()
|
||||
}
|
||||
|
||||
data class Genre(val id: Int, val name: String) {
|
||||
companion object {
|
||||
val COMEDY = Genre(4, "Comedy")
|
||||
val DEMONS = Genre(6, "Demons")
|
||||
val DRAMA = Genre(8, "Drama")
|
||||
val FANTASY = Genre(10, "Fantasy")
|
||||
val HENTAI = Genre(12, "Hentai")
|
||||
val MAGIC = Genre(16, "Magic")
|
||||
val PARODY = Genre(20, "Parody")
|
||||
val ROMANCE = Genre(22, "Romance")
|
||||
val SCHOOL = Genre(23, "School")
|
||||
val SCIFI = Genre(24, "Sci-Fi")
|
||||
val SHOUJO = Genre(25, "Shoujo")
|
||||
val HAREM = Genre(35, "Harem")
|
||||
val MILITARY = Genre(38, "Military")
|
||||
}
|
||||
}
|
||||
|
||||
fun getRelated(): List<Relation> {
|
||||
return dom.select("table.anime_detail_related_anime tr").flatMap {
|
||||
val type = it.child(0).text().trim().replace(":", "")
|
||||
it.child(1).select("a").mapNotNull { el ->
|
||||
el.attr("href")
|
||||
?.let { href ->
|
||||
val parts = href.split("/")
|
||||
|
||||
if (parts.contains("anime")) {
|
||||
parts[parts.indexOf("anime") + 1].toLongOrNull()
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
?.let { id -> Relation(type, id) }
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getProducers(): List<ProducerRelation> {
|
||||
fun Pair<String, Element>?.getProducers(): List<Producer> {
|
||||
return this?.second?.let {
|
||||
it.select("a").mapNotNull { a ->
|
||||
val href = a.attr("href").split("/")
|
||||
if (href.contains("producer")) Producer(
|
||||
href[href.indexOf("producer") + 1].toInt(),
|
||||
a.text().trim()
|
||||
) else null
|
||||
}
|
||||
} ?: emptyList()
|
||||
}
|
||||
|
||||
val producers = items["producers"].getProducers()
|
||||
val studios = items["studios"].getProducers()
|
||||
val licensors = items["licensors"].getProducers()
|
||||
|
||||
return producers.map { ProducerRelation(ProducerFunction.Producer, it) } +
|
||||
studios.map { ProducerRelation(ProducerFunction.Studio, it) } +
|
||||
licensors.map { ProducerRelation(ProducerFunction.Licensor, it) }
|
||||
}
|
||||
|
||||
data class ProducerRelation(val function: ProducerFunction, val producer: Producer)
|
||||
|
||||
data class Producer(val id: Int, val name: String)
|
||||
|
||||
fun getRating(): MyAnimeListInfo.Rating? {
|
||||
return MyAnimeListInfo.Rating.fromString(items["rating"]?.first?.split(" - ")?.first()?.trim() ?: return null)
|
||||
}
|
||||
|
||||
private val durationRegex = Regex("(?:(\\d+) hrs.)?(?:(\\d+) min.)?")
|
||||
|
||||
fun getDuration(): Duration? {
|
||||
val txt = items["duration"]?.first?.split(" per ", limit = 2)?.firstOrNull() ?: return null
|
||||
val grps = durationRegex.matchEntire(txt)?.groups ?: return null
|
||||
val hrs = grps[1]
|
||||
val min = grps[2]
|
||||
var x = Duration.ofMillis(0)
|
||||
x += Duration.ofHours(hrs?.value?.toInt() ?: 0)
|
||||
x += Duration.ofMinutes(min?.value?.toInt() ?: 0)
|
||||
return x
|
||||
}
|
||||
|
||||
|
||||
data class Relation(val type: AnimeRelation.RelationType, val id: Long) {
|
||||
constructor(type: String, id: Long) : this(AnimeRelation.RelationType.fromString(type), id)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,178 @@
|
||||
package moe.odango.index.sync
|
||||
|
||||
import com.github.kittinunf.fuel.coroutines.awaitByteArrayResponseResult
|
||||
import com.github.kittinunf.fuel.httpDownload
|
||||
import io.requery.Persistable
|
||||
import io.requery.kotlin.`in`
|
||||
import io.requery.kotlin.eq
|
||||
import io.requery.kotlin.invoke
|
||||
import io.requery.sql.KotlinEntityDataStore
|
||||
import moe.odango.index.di
|
||||
import moe.odango.index.entity.Anime
|
||||
import moe.odango.index.entity.Title
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.XMLOutputStreamReader
|
||||
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
|
||||
import org.kodein.di.instance
|
||||
import java.io.File
|
||||
import java.io.InputStream
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.zip.InflaterOutputStream
|
||||
|
||||
class AniDBTitleSync : ScheduledSync(1, TimeUnit.DAYS) {
|
||||
private val entityStore by di.instance<KotlinEntityDataStore<Persistable>>()
|
||||
|
||||
data class AniDBTitle(val aid: Long, val name: String, val type: String, val language: String)
|
||||
|
||||
override suspend fun run() {
|
||||
|
||||
|
||||
val (feeder, cacheMap) = getFeeder()
|
||||
|
||||
"https://anidb.net/api/anime-titles.xml.gz"
|
||||
.httpDownload()
|
||||
.streamDestination { _, _ ->
|
||||
InflaterOutputStream(feeder) to { InputStream.nullInputStream() }
|
||||
}
|
||||
.awaitByteArrayResponseResult()
|
||||
|
||||
syncTitles(cacheMap)
|
||||
}
|
||||
|
||||
private fun getFeeder(): Pair<XMLOutputStreamReader, MutableMap<Long, MutableList<AniDBTitle>>> {
|
||||
var animeId: Long = 0;
|
||||
var language = ""
|
||||
var type = ""
|
||||
var title = ""
|
||||
val cacheMap = mutableMapOf<Long, MutableList<AniDBTitle>>()
|
||||
|
||||
return XMLOutputStreamReader {
|
||||
if (isStartElement) {
|
||||
if (name.localPart == "anime") {
|
||||
animeId = getAttributeValue("", "aid").toLong()
|
||||
}
|
||||
|
||||
if (name.localPart == "title") {
|
||||
title = ""
|
||||
type = getAttributeValue("", "type")
|
||||
language = getAttributeValue("http://www.w3.org/XML/1998/namespace", "lang")
|
||||
}
|
||||
}
|
||||
|
||||
if (isCharacters) {
|
||||
title += text
|
||||
}
|
||||
|
||||
if (isEndElement) {
|
||||
if (name.localPart == "title") {
|
||||
cacheMap
|
||||
.getOrPut(animeId, ::mutableListOf)
|
||||
.add(AniDBTitle(animeId, title, type, language))
|
||||
|
||||
println("$animeId => [$type/$language] $title")
|
||||
|
||||
title = ""
|
||||
}
|
||||
|
||||
if (name.localPart == "anime") {
|
||||
if (cacheMap.size >= 100) {
|
||||
syncTitles(cacheMap)
|
||||
|
||||
cacheMap.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
} to cacheMap
|
||||
}
|
||||
|
||||
fun syncWithFile(file: File) {
|
||||
val (feeder, cacheMap) = getFeeder()
|
||||
|
||||
file
|
||||
.inputStream()
|
||||
.let {
|
||||
if (file.name.endsWith(".gz")) {
|
||||
GzipCompressorInputStream(it)
|
||||
} else {
|
||||
it
|
||||
}
|
||||
}
|
||||
.transferTo(feeder)
|
||||
syncTitles(cacheMap)
|
||||
}
|
||||
|
||||
private fun syncTitles(cacheMap: Map<Long, List<AniDBTitle>>) {
|
||||
val ids = cacheMap.keys
|
||||
|
||||
val (animes, titles) = entityStore {
|
||||
val animes = (select(Anime::class) where (Anime::aniDbId `in` ids)).get().toList()
|
||||
|
||||
val query = select(Title::class) where (Title::anime `in` animes) and (Title::source eq InfoSource.AniDB)
|
||||
animes to query().toList()
|
||||
}
|
||||
|
||||
val animeById = animes.associateBy { it.aniDbId!! }
|
||||
val titlesByAniDBId = titles.groupBy {
|
||||
it.anime.aniDbId!!
|
||||
}
|
||||
|
||||
val newAnimes = mutableListOf<Pair<Long, List<AniDBTitle>>>()
|
||||
|
||||
entityStore.withTransaction {
|
||||
for ((aniDb, newTitles) in cacheMap) {
|
||||
val newAniDbTitlesSet = newTitles.map { Triple(it.language, it.type, it.name) }.toMutableSet()
|
||||
val anime = animeById[aniDb]
|
||||
if (anime == null) {
|
||||
newAnimes.add(aniDb to newTitles)
|
||||
continue
|
||||
}
|
||||
|
||||
val dbTitles = titlesByAniDBId[aniDb] ?: listOf()
|
||||
|
||||
for (dbTitle in dbTitles) {
|
||||
if (!newAniDbTitlesSet.remove(
|
||||
Triple(
|
||||
dbTitle.language,
|
||||
dbTitle.type.toAniDBString(),
|
||||
dbTitle.name
|
||||
)
|
||||
)
|
||||
) {
|
||||
// If it wasn't in the set, remove it from the DB
|
||||
delete(dbTitle)
|
||||
}
|
||||
}
|
||||
|
||||
for ((language, type, name) in newAniDbTitlesSet) {
|
||||
val title = Title {
|
||||
setAnime(anime)
|
||||
setLanguage(language)
|
||||
setType(Title.TitleType.fromAniDBString(type))
|
||||
setSource(InfoSource.AniDB)
|
||||
this.name = name
|
||||
}
|
||||
|
||||
insert(title)
|
||||
}
|
||||
}
|
||||
|
||||
for ((aniDb, aniDbTitles) in newAnimes) {
|
||||
val anime = Anime {
|
||||
aniDbId = aniDb
|
||||
}
|
||||
|
||||
insert(anime)
|
||||
|
||||
for (aniDbTitle in aniDbTitles) {
|
||||
insert(Title {
|
||||
setAnime(anime)
|
||||
setLanguage(aniDbTitle.language)
|
||||
setType(Title.TitleType.fromAniDBString(aniDbTitle.type))
|
||||
setSource(InfoSource.AniDB)
|
||||
name = aniDbTitle.name
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,318 @@
|
||||
package moe.odango.index.sync
|
||||
|
||||
import io.requery.Persistable
|
||||
import io.requery.kotlin.`in`
|
||||
import io.requery.kotlin.eq
|
||||
import io.requery.kotlin.invoke
|
||||
import io.requery.sql.KotlinEntityDataStore
|
||||
import moe.odango.index.di
|
||||
import moe.odango.index.entity.*
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import moe.odango.index.utils.MergeMap
|
||||
import moe.odango.index.utils.brText
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Element
|
||||
import org.jsoup.parser.Parser
|
||||
import org.kodein.di.instance
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
|
||||
class AniDBXMLSync {
|
||||
private val entityStore by di.instance<KotlinEntityDataStore<Persistable>>()
|
||||
private val seriesToMerge = MergeMap<UUID>()
|
||||
|
||||
fun run(xmlDir: String) {
|
||||
val files = File(xmlDir)
|
||||
.listFiles()
|
||||
.filter {
|
||||
it.extension == "xml"
|
||||
}
|
||||
|
||||
for (file in files) {
|
||||
val xml = file.readText(Charsets.UTF_8)
|
||||
indexAniDBEntry(xml)
|
||||
}
|
||||
}
|
||||
|
||||
private fun indexAniDBEntry(xml: String) {
|
||||
val doc = Jsoup.parse(xml, "", Parser.xmlParser())
|
||||
val anime = doc.selectFirst("anime")
|
||||
val aid = anime.attr("id").toLongOrNull() ?: return
|
||||
|
||||
val animeEnt: Anime? = entityStore {
|
||||
val q = select(Anime::class) where (Anime::aniDbId eq aid)
|
||||
q().firstOrNull()
|
||||
}
|
||||
|
||||
val myAnimeListId = anime
|
||||
.selectFirst("resource[type=2] externalentity identifier")
|
||||
?.text()
|
||||
?.toLongOrNull()
|
||||
|
||||
val malAnimeEnt = myAnimeListId?.let {
|
||||
entityStore {
|
||||
val q = select(Anime::class) where (Anime::myAnimeListId eq myAnimeListId)
|
||||
|
||||
q().firstOrNull()
|
||||
}
|
||||
}
|
||||
|
||||
if (malAnimeEnt != null && animeEnt != null) {
|
||||
// WE GOT ISSUES OH NO
|
||||
TODO()
|
||||
}
|
||||
|
||||
val entity = malAnimeEnt ?: animeEnt ?: Anime {
|
||||
this.myAnimeListId = myAnimeListId
|
||||
this.aniDbId = aid
|
||||
}.let { entityStore.insert(it) }
|
||||
|
||||
val aniDbInfo = entity.aniDbInfo ?: AniDBInfo {
|
||||
setAnime(entity)
|
||||
}.let { entityStore.insert(it) }
|
||||
|
||||
aniDbInfo.restricted = anime.attr("restricted") == "true"
|
||||
aniDbInfo.description = doc.selectFirst("anime > description")?.brText() ?: ""
|
||||
aniDbInfo.episodes = doc.selectFirst("anime > episodecount")?.text()?.trim()?.toIntOrNull()
|
||||
aniDbInfo.type =
|
||||
doc.selectFirst("anime > type")?.text()?.trim()?.let { AniDBInfo.ReleaseType.fromAniDBString(it) }
|
||||
?: AniDBInfo.ReleaseType.Unknown
|
||||
aniDbInfo.image = doc.selectFirst("anime > picture")?.text()?.trim()
|
||||
val startDate = doc.selectFirst("anime > startdate")?.text()?.trim()?.let {
|
||||
val items = it.split("-")
|
||||
if (items.size > 3) {
|
||||
null
|
||||
} else {
|
||||
items.map { item -> item.toIntOrNull() }
|
||||
}
|
||||
} ?: emptyList()
|
||||
|
||||
aniDbInfo.startYear = startDate[0]
|
||||
aniDbInfo.startMonth = startDate[1]
|
||||
aniDbInfo.startDay = startDate[2]
|
||||
|
||||
val endDate = doc.selectFirst("anime > enddate")?.text()?.trim()?.let {
|
||||
val items = it.split("-")
|
||||
if (items.size > 3) {
|
||||
null
|
||||
} else {
|
||||
items.map { item -> item.toIntOrNull() }
|
||||
}
|
||||
} ?: emptyList()
|
||||
|
||||
aniDbInfo.endYear = endDate[0]
|
||||
aniDbInfo.endMonth = endDate[1]
|
||||
aniDbInfo.endDay = endDate[2]
|
||||
|
||||
entityStore.update(aniDbInfo)
|
||||
|
||||
val aniDbTitles = entityStore {
|
||||
val q = select(Title::class) where (Title::anime eq entity) and (Title::source eq InfoSource.AniDB)
|
||||
q().toList()
|
||||
}
|
||||
|
||||
.associateBy { "${it.type}/${it.language}/${it.name}" }
|
||||
.toMutableMap()
|
||||
|
||||
for (title in doc.select("anime > titles > title")) {
|
||||
val name = title.text() ?: continue
|
||||
val lang = title.attr("lang") ?: null
|
||||
val type = title.attr("type")?.let { Title.TitleType.fromAniDBString(it) } ?: Title.TitleType.Synonym
|
||||
val handle = "$type/$lang/$name"
|
||||
|
||||
if (aniDbTitles.remove(handle) == null) {
|
||||
entityStore.insert(Title {
|
||||
setAnime(entity)
|
||||
setSource(InfoSource.AniDB)
|
||||
setLanguage(lang)
|
||||
setType(type)
|
||||
|
||||
this.name = name
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for ((_, title) in aniDbTitles) {
|
||||
entityStore.delete(title)
|
||||
}
|
||||
|
||||
val relatedAnimeFrom = entityStore {
|
||||
val q =
|
||||
select(AnimeRelation::class) where (AnimeRelation::from eq entity) and (AnimeRelation::source eq InfoSource.AniDB)
|
||||
q().toList()
|
||||
}
|
||||
.associateBy { it.to.aniDbId!! }
|
||||
.toMutableMap()
|
||||
|
||||
val relatedAnimeTo = entityStore {
|
||||
val q =
|
||||
select(AnimeRelation::class) where (AnimeRelation::to eq entity) and (AnimeRelation::source eq InfoSource.AniDB)
|
||||
q().toList()
|
||||
}
|
||||
.associateBy { it.from.aniDbId!! }
|
||||
.toMutableMap()
|
||||
|
||||
for (related in doc.select("anime > relatedanime > anime")) {
|
||||
val id = related.attr("id").toLongOrNull() ?: continue
|
||||
val relatedAnime = entityStore {
|
||||
val q = select(Anime::class) where (Anime::aniDbId eq id)
|
||||
q().firstOrNull()
|
||||
} ?: Anime { aniDbId = id }.let { entityStore.insert(it) }
|
||||
|
||||
val type = related.attr("type")?.let { AnimeRelation.RelationType.fromString(it) }
|
||||
?: AnimeRelation.RelationType.Other
|
||||
|
||||
val currentFromRelation = relatedAnimeFrom.remove(id)
|
||||
if (currentFromRelation == null) {
|
||||
entityStore.insert(AnimeRelation {
|
||||
setFrom(entity)
|
||||
setTo(relatedAnime)
|
||||
setSource(InfoSource.AniDB)
|
||||
relation = type
|
||||
})
|
||||
} else if (currentFromRelation.relation != type) {
|
||||
currentFromRelation.relation = type
|
||||
entityStore.update(currentFromRelation)
|
||||
}
|
||||
|
||||
val currentToRelation = relatedAnimeTo.remove(id)
|
||||
if (currentToRelation == null) {
|
||||
entityStore.insert(AnimeRelation {
|
||||
setFrom(relatedAnime)
|
||||
setTo(entity)
|
||||
setSource(InfoSource.AniDB)
|
||||
relation = type.inverse
|
||||
})
|
||||
}
|
||||
|
||||
// Character only have characters of that anime in the other anime
|
||||
// So are not part of The Series, see e.g. Isekai Quartet
|
||||
if (type != AnimeRelation.RelationType.Character) {
|
||||
if (relatedAnime.series != null && entity.series == null) {
|
||||
entity.series = relatedAnime.series
|
||||
entityStore.update(entity)
|
||||
} else if (relatedAnime.series == null && entity.series != null) {
|
||||
relatedAnime.series = entity.series
|
||||
entityStore.update(relatedAnime)
|
||||
} else if (relatedAnime.series == null && entity.series == null) {
|
||||
val newSeries = AnimeSeries {}
|
||||
entityStore.insert(newSeries)
|
||||
entity.series = newSeries
|
||||
entityStore.update(entity)
|
||||
relatedAnime.series = newSeries
|
||||
entityStore.update(relatedAnime)
|
||||
} else if (relatedAnime.series != null && entity.series != null) {
|
||||
seriesToMerge.add(relatedAnime.series!!.id, entity.series!!.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val tags = entityStore {
|
||||
val q =
|
||||
select(AnimeTag::class) join Tag::class on (AnimeTag::tag eq Tag::class) where (AnimeTag::anime eq entity) and (AnimeTag::source eq InfoSource.AniDB)
|
||||
q().toList()
|
||||
}
|
||||
.associateBy { it.tag.aniDbId!! }
|
||||
.toMutableMap()
|
||||
|
||||
val tagIds = doc.select("anime > tags > tag").flatMap {
|
||||
val ids = mutableSetOf<Long>()
|
||||
it.attr("id")?.toLongOrNull()?.let(ids::add)
|
||||
it.attr("parentid")?.toLongOrNull()?.let(ids::add)
|
||||
|
||||
ids
|
||||
}
|
||||
.toSet()
|
||||
|
||||
val tagsById = entityStore {
|
||||
val q = select(Tag::class) where (Tag::aniDbId `in` tagIds)
|
||||
q().toList()
|
||||
}
|
||||
.associateBy { it.aniDbId!! }
|
||||
.toMutableMap()
|
||||
|
||||
val after = mutableListOf<Element>()
|
||||
|
||||
fun getTag(tag: Element): Tag {
|
||||
val tagId = tag.attr("id")?.toLongOrNull()!!
|
||||
val name = tag.selectFirst("name")?.text()?.trim()!!
|
||||
val description = tag.selectFirst("description")?.brText() ?: ""
|
||||
val parentId = tag.attr("parentid")?.toLongOrNull()
|
||||
|
||||
return tagsById.getOrPut(tagId) {
|
||||
val newTag = Tag {
|
||||
setAniDbId(tagId)
|
||||
setName(name)
|
||||
setDescription(description)
|
||||
setSpoiler(tag.attr("globalspoiler") == "true")
|
||||
parentId?.let {
|
||||
setParentId(tagsById[it]!!.id)
|
||||
}
|
||||
}
|
||||
|
||||
entityStore.insert(newTag)
|
||||
newTag
|
||||
}
|
||||
}
|
||||
|
||||
fun ensureTag(tag: Element, tagId: Long) {
|
||||
if (tags.remove(tagId) == null) {
|
||||
val tagEntity = getTag(tag)
|
||||
val animeTag = AnimeTag {
|
||||
setTag(tagEntity)
|
||||
setAnime(entity)
|
||||
setSpoiler(tag.attr("localspoiler") == "true")
|
||||
setSource(InfoSource.AniDB)
|
||||
}
|
||||
|
||||
entityStore.insert(animeTag)
|
||||
}
|
||||
}
|
||||
|
||||
for (tag in doc.select("anime > tags > tag")) {
|
||||
val tagId = tag.attr("id")?.toLongOrNull() ?: continue
|
||||
val name = tag.selectFirst("name")?.text()?.trim() ?: continue
|
||||
val parentId = tag.attr("parentid")?.toLongOrNull()
|
||||
|
||||
if (parentId != null && !tagsById.contains(parentId)) {
|
||||
after.add(tag)
|
||||
continue
|
||||
}
|
||||
|
||||
ensureTag(tag, tagId)
|
||||
}
|
||||
|
||||
for (tag in after) {
|
||||
val tagId = tag.attr("id")?.toLongOrNull() ?: continue
|
||||
ensureTag(tag, tagId)
|
||||
}
|
||||
|
||||
for ((_, tag) in tags) {
|
||||
entityStore.delete(tag)
|
||||
}
|
||||
}
|
||||
|
||||
fun mergeSeries() {
|
||||
for (coll in seriesToMerge) {
|
||||
val items = coll.toMutableSet()
|
||||
val first = coll.first()
|
||||
items.remove(first)
|
||||
|
||||
entityStore {
|
||||
val u = update(AnimeSeries::class)
|
||||
.set(AnimeSeriesEntity.REPLACED_WITH, first)
|
||||
.where((AnimeSeries::id `in` items) or (AnimeSeries::replacedWith `in` items))
|
||||
|
||||
u()
|
||||
|
||||
val u2 = update(Anime::class)
|
||||
.set(AnimeEntity.SERIES_ID, first)
|
||||
.where(Anime::series `in` items)
|
||||
|
||||
u2()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,122 @@
|
||||
package moe.odango.index.sync
|
||||
|
||||
import com.github.kittinunf.fuel.coroutines.awaitStringResult
|
||||
import com.github.kittinunf.fuel.httpGet
|
||||
import io.requery.Persistable
|
||||
import io.requery.kotlin.`in`
|
||||
import io.requery.kotlin.eq
|
||||
import io.requery.kotlin.invoke
|
||||
import io.requery.sql.KotlinEntityDataStore
|
||||
import moe.odango.index.di
|
||||
import moe.odango.index.entity.Anime
|
||||
import moe.odango.index.entity.MyAnimeListInfo
|
||||
import moe.odango.index.entity.Title
|
||||
import moe.odango.index.scraper.mal.AnimeListScraper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import org.kodein.di.instance
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class MyAnimeListListingSync : ScheduledSync(2, TimeUnit.DAYS) {
|
||||
val entityStore: KotlinEntityDataStore<Persistable> by di.instance()
|
||||
|
||||
override suspend fun run() {
|
||||
var offset = 0
|
||||
|
||||
do {
|
||||
println(" => MAL Offset $offset")
|
||||
|
||||
val items = getListing(offset)
|
||||
val animes = entityStore {
|
||||
val q = select(Anime::class) where (Anime::myAnimeListId `in` items.map { it.myAnimeListId })
|
||||
|
||||
q().toList()
|
||||
}.associateBy {
|
||||
it.myAnimeListId!!
|
||||
}
|
||||
|
||||
val infos = entityStore {
|
||||
val q =
|
||||
select(MyAnimeListInfo::class) join Anime::class on (MyAnimeListInfo::anime `in` animes.values)
|
||||
q().toList()
|
||||
}.associateBy {
|
||||
it.anime.myAnimeListId!!
|
||||
}
|
||||
|
||||
val titles = entityStore {
|
||||
val q =
|
||||
select(Title::class) where (Title::source eq InfoSource.MyAnimeList) and (Title::language eq "en") and (Title::anime `in` animes.values)
|
||||
|
||||
q().toList()
|
||||
}.associateBy {
|
||||
it.anime.myAnimeListId!!
|
||||
}
|
||||
|
||||
entityStore.withTransaction {
|
||||
for (item in items) {
|
||||
println("Syncing MAL#${item.myAnimeListId} ${item.title}")
|
||||
val anime = animes[item.myAnimeListId] ?: run {
|
||||
val newAnime = Anime {
|
||||
myAnimeListId = item.myAnimeListId
|
||||
}
|
||||
|
||||
insert(newAnime)
|
||||
|
||||
newAnime
|
||||
}
|
||||
|
||||
val title = titles[item.myAnimeListId] ?: run {
|
||||
val newTitle = Title {
|
||||
setAnime(anime)
|
||||
setLanguage("en")
|
||||
setType(Title.TitleType.Official)
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
name = item.title
|
||||
}
|
||||
|
||||
insert(newTitle)
|
||||
|
||||
newTitle
|
||||
}
|
||||
|
||||
if (title.name != item.title) {
|
||||
title.name = item.title
|
||||
|
||||
update(title)
|
||||
}
|
||||
|
||||
val info = infos[item.myAnimeListId] ?: run {
|
||||
val newInfo = MyAnimeListInfo {
|
||||
setAnime(anime)
|
||||
episodes = item.episodes
|
||||
releaseType = item.type
|
||||
}
|
||||
|
||||
insert(newInfo)
|
||||
|
||||
newInfo
|
||||
}
|
||||
|
||||
if (info.episodes != item.episodes || info.releaseType != item.type) {
|
||||
info.episodes = item.episodes
|
||||
info.releaseType = item.type
|
||||
|
||||
update(info)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
offset += items.size
|
||||
} while (items.size == 50)
|
||||
}
|
||||
|
||||
suspend fun getListing(offset: Int): List<AnimeListScraper.MyAnimeListListingItem> {
|
||||
val url = "https://myanimelist.net/anime.php?o=9&c[0]=a&c[1]=b&cv=2&w=1&show=$offset"
|
||||
|
||||
val body = url
|
||||
.httpGet()
|
||||
.awaitStringResult()
|
||||
.get()
|
||||
|
||||
return AnimeListScraper(body).getItems()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,327 @@
|
||||
package moe.odango.index.sync
|
||||
|
||||
import com.github.kittinunf.fuel.coroutines.awaitString
|
||||
import com.github.kittinunf.fuel.httpGet
|
||||
import io.requery.Persistable
|
||||
import io.requery.kotlin.*
|
||||
import io.requery.query.function.Random
|
||||
import io.requery.sql.KotlinEntityDataStore
|
||||
import moe.odango.index.di
|
||||
import moe.odango.index.entity.*
|
||||
import moe.odango.index.scraper.mal.AnimePageScraper
|
||||
import moe.odango.index.utils.InfoSource
|
||||
import org.joda.time.DateTime
|
||||
import org.kodein.di.instance
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.max
|
||||
|
||||
class MyAnimeListPageSync : ScheduledSync(1, TimeUnit.DAYS) {
|
||||
val entityStore: KotlinEntityDataStore<Persistable> by di.instance()
|
||||
|
||||
override suspend fun run() {
|
||||
val weekAgo = DateTime((Instant.now() - Duration.ofDays(7)).toEpochMilli())
|
||||
val infos = entityStore {
|
||||
val q =
|
||||
select(MyAnimeListInfo::class) where (MyAnimeListInfo::lastScrape lt weekAgo) or (MyAnimeListInfo::lastScrape.isNull()) orderBy (Random()) limit 300
|
||||
q().toList()
|
||||
}
|
||||
|
||||
val animes = infos.map { it.anime }
|
||||
val animeById = animes.associateBy { it.id }
|
||||
|
||||
val titles = entityStore {
|
||||
val q = select(Title::class) where (Title::anime `in` animes) and (Title::source eq InfoSource.MyAnimeList)
|
||||
q().toList()
|
||||
}
|
||||
|
||||
val genres = entityStore {
|
||||
val q =
|
||||
select(AnimeGenre::class) where (AnimeGenre::anime `in` animes) and (AnimeGenre::source eq InfoSource.MyAnimeList)
|
||||
q().toList()
|
||||
}.groupBy {
|
||||
it.anime.id
|
||||
}.mapValues { entry ->
|
||||
entry.value.associateBy { it.genre.myAnimeListId }.toMutableMap()
|
||||
}
|
||||
|
||||
val fromRelations = entityStore {
|
||||
val q =
|
||||
select(AnimeRelation::class) where (AnimeRelation::from `in` animes) and (AnimeRelation::source eq InfoSource.MyAnimeList)
|
||||
q().toList()
|
||||
}
|
||||
.groupBy { it.from.id }
|
||||
.mapValues { it.value.associateBy { it.from.myAnimeListId!! }.toMutableMap() }
|
||||
|
||||
val toRelations = entityStore {
|
||||
val q =
|
||||
select(AnimeRelation::class) where (AnimeRelation::source eq InfoSource.MyAnimeList) and (AnimeRelation::to `in` animes)
|
||||
q().toList()
|
||||
}
|
||||
.groupBy { it.to.id }
|
||||
.mapValues { it.value.associateBy { it.to.myAnimeListId!! }.toMutableMap() }
|
||||
|
||||
val producers = entityStore {
|
||||
val q =
|
||||
select(AnimeProducer::class) where (AnimeProducer::anime `in` animes) and (AnimeProducer::source eq InfoSource.MyAnimeList)
|
||||
q().toList()
|
||||
}
|
||||
.groupBy { it.anime.id }
|
||||
.mapValues { it.value.associateBy { it.producer.myAnimeListId!! }.toMutableMap() }
|
||||
|
||||
val titlesByMyAnimeListId = mutableMapOf<Long, MutableMap<String, Title>>()
|
||||
|
||||
for (title in titles) {
|
||||
titlesByMyAnimeListId
|
||||
.getOrPut(title.anime.myAnimeListId!!, ::mutableMapOf)[title.name] = title
|
||||
}
|
||||
|
||||
val bodies = mutableMapOf<Long, String>()
|
||||
|
||||
for (info in infos) {
|
||||
try {
|
||||
val myAnimeListId = info.anime.myAnimeListId!!
|
||||
println("=> Fetching MAL Page: $myAnimeListId")
|
||||
val body = "https://myanimelist.net/anime/$myAnimeListId/"
|
||||
.httpGet()
|
||||
.awaitString()
|
||||
|
||||
bodies[myAnimeListId] = body
|
||||
} catch (t: Throwable) {
|
||||
t.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
val allGenres = mutableMapOf<Int, Genre>()
|
||||
val allProducers = mutableMapOf<Int, Producer>()
|
||||
val seriesToMerge = mutableListOf<Pair<AnimeSeries, AnimeSeries>>()
|
||||
|
||||
entityStore.withTransaction {
|
||||
fun getGenre(genre: AnimePageScraper.Genre): Genre {
|
||||
return allGenres.getOrPut(genre.id) {
|
||||
val gq = select(Genre::class) where (Genre::myAnimeListId eq genre.id)
|
||||
gq().firstOrNull() ?: run {
|
||||
val gen = Genre {
|
||||
setMyAnimeListId(genre.id)
|
||||
setName(genre.name)
|
||||
}
|
||||
|
||||
insert(gen)
|
||||
|
||||
gen
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getProducer(producer: AnimePageScraper.Producer): Producer {
|
||||
return allProducers.getOrPut(producer.id) {
|
||||
val pq = select(Producer::class) where (Producer::myAnimeListId eq producer.id)
|
||||
pq().firstOrNull() ?: run {
|
||||
val prod = Producer {
|
||||
setMyAnimeListId(producer.id)
|
||||
setName(producer.name)
|
||||
}
|
||||
|
||||
insert(prod)
|
||||
|
||||
prod
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (info in infos) {
|
||||
val myAnimeListId = info.anime.myAnimeListId!!
|
||||
println("=> Indexing MAL Page: $myAnimeListId")
|
||||
|
||||
val body = bodies[myAnimeListId] ?: continue
|
||||
val currentTitles = titlesByMyAnimeListId[myAnimeListId] ?: mutableMapOf()
|
||||
|
||||
val scraper = AnimePageScraper(body)
|
||||
val aired = scraper.getAired()
|
||||
val premiered = scraper.getPremiered()
|
||||
|
||||
info.airedEnd = aired?.end?.toDate()
|
||||
info.airedStart = aired?.start?.toDate()
|
||||
info.premieredSeason = premiered?.season?.name
|
||||
info.premieredYear = premiered?.year
|
||||
info.releaseType = scraper.getReleaseType()
|
||||
info.image = scraper.getImage()
|
||||
info.source = scraper.getSource()
|
||||
info.description = scraper.getDescription()
|
||||
info.episodes = scraper.getEpisodes()
|
||||
info.rating = scraper.getRating()
|
||||
info.duration = scraper.getDuration()?.toSeconds()?.toInt()?.let { max(it, 0) }
|
||||
info.lastScrape = DateTime.now()
|
||||
|
||||
update(info)
|
||||
|
||||
val done = mutableSetOf<String>()
|
||||
|
||||
val title = scraper.getTitle()
|
||||
if (currentTitles.remove(title) == null && !done.contains(title)) {
|
||||
insert(Title {
|
||||
setAnime(info.anime)
|
||||
name = title
|
||||
setType(Title.TitleType.Main)
|
||||
setLanguage("x-jat")
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
})
|
||||
}
|
||||
done.add(title)
|
||||
|
||||
val englishName = scraper.getEnglishName()
|
||||
if (englishName != null && currentTitles.remove(englishName) == null && !done.contains(englishName)) {
|
||||
insert(Title {
|
||||
setAnime(info.anime)
|
||||
name = englishName
|
||||
setType(Title.TitleType.Official)
|
||||
setLanguage("en")
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
})
|
||||
}
|
||||
englishName?.let(done::add)
|
||||
|
||||
val japaneseName = scraper.getJapaneseName()
|
||||
if (japaneseName != null && currentTitles.remove(japaneseName) == null && !done.contains(japaneseName)) {
|
||||
insert(Title {
|
||||
setAnime(info.anime)
|
||||
name = japaneseName
|
||||
setType(Title.TitleType.Official)
|
||||
setLanguage("ja")
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
})
|
||||
}
|
||||
|
||||
japaneseName?.let(done::add)
|
||||
|
||||
val synonyms = scraper.getSynonyms()
|
||||
for (synonym in synonyms) {
|
||||
if (currentTitles.remove(synonym) == null && !done.contains(synonym)) {
|
||||
insert(Title {
|
||||
setAnime(info.anime)
|
||||
name = synonym
|
||||
setType(Title.TitleType.Synonym)
|
||||
setLanguage("x-jat")
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
})
|
||||
}
|
||||
done.add(synonym)
|
||||
}
|
||||
|
||||
for ((_, currentTitle) in currentTitles) {
|
||||
delete(currentTitle)
|
||||
}
|
||||
|
||||
val currentGenres = genres[info.anime.id] ?: mutableMapOf()
|
||||
|
||||
for (genre in scraper.getGenres()) {
|
||||
if (currentGenres.remove(genre.id) == null) {
|
||||
val genreEnt = getGenre(genre)
|
||||
insert(AnimeGenre {
|
||||
setAnime(info.anime)
|
||||
setGenre(genreEnt)
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for ((_, genreEnt) in currentGenres) {
|
||||
delete(genreEnt)
|
||||
}
|
||||
|
||||
val currentFromRelations = fromRelations[info.anime.id] ?: mutableMapOf()
|
||||
val currentToRelations = toRelations[info.anime.id] ?: mutableMapOf()
|
||||
|
||||
val related = scraper.getRelated()
|
||||
val relatedAnimesQuery = select(Anime::class) where (Anime::myAnimeListId `in` related.map { it.id })
|
||||
val relatedAnimes = relatedAnimesQuery().toList().associateBy { it.myAnimeListId!! }.toMutableMap()
|
||||
|
||||
for (relation in related) {
|
||||
val currentFromRelation = currentFromRelations.remove(relation.id)
|
||||
val relatedAnime = relatedAnimes.getOrPut(relation.id) {
|
||||
val anim = Anime {
|
||||
this@Anime.myAnimeListId = relation.id
|
||||
}
|
||||
|
||||
insert(anim)
|
||||
|
||||
anim
|
||||
}
|
||||
if (currentFromRelation == null) {
|
||||
insert(AnimeRelation {
|
||||
setFrom(info.anime)
|
||||
setTo(relatedAnime)
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
this@AnimeRelation.relation = relation.type
|
||||
})
|
||||
} else if (currentFromRelation.relation != relation.type) {
|
||||
currentFromRelation.relation = relation.type
|
||||
update(currentFromRelation)
|
||||
}
|
||||
|
||||
val currentToRelation = currentToRelations.remove(relation.id)
|
||||
if (currentToRelation == null) {
|
||||
insert(AnimeRelation {
|
||||
setTo(info.anime)
|
||||
setFrom(relatedAnime)
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
this@AnimeRelation.relation = relation.type.inverse
|
||||
})
|
||||
}
|
||||
|
||||
// Character only have characters of that anime in the other anime
|
||||
// So are not part of The Series, see e.g. Isekai Quartet
|
||||
if (relation.type != AnimeRelation.RelationType.Character) {
|
||||
val currAnime = info.anime
|
||||
if (relatedAnime.series != null && currAnime.series == null) {
|
||||
currAnime.series = relatedAnime.series
|
||||
update(currAnime)
|
||||
} else if (relatedAnime.series == null && currAnime.series != null) {
|
||||
relatedAnime.series = currAnime.series
|
||||
update(relatedAnime)
|
||||
} else if (relatedAnime.series == null && currAnime.series == null) {
|
||||
val newSeries = AnimeSeries {}
|
||||
insert(newSeries)
|
||||
currAnime.series = newSeries
|
||||
update(currAnime)
|
||||
relatedAnime.series = newSeries
|
||||
update(relatedAnime)
|
||||
} else if (relatedAnime.series != null && currAnime.series != null) {
|
||||
seriesToMerge.add(relatedAnime.series!! to currAnime.series!!)
|
||||
}
|
||||
}
|
||||
|
||||
val currentProducers = producers[info.anime.id] ?: mutableMapOf()
|
||||
|
||||
val malProducers = scraper.getProducers()
|
||||
for (producer in malProducers) {
|
||||
val currentProducer = currentProducers.remove(producer.producer.id)
|
||||
if (currentProducer != null) {
|
||||
if (currentProducer.function != producer.function) {
|
||||
currentProducer.function = producer.function
|
||||
update(currentProducer)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
val producerEntity = getProducer(producer.producer)
|
||||
|
||||
insert(AnimeProducer {
|
||||
setAnime(info.anime)
|
||||
setProducer(producerEntity)
|
||||
setSource(InfoSource.MyAnimeList)
|
||||
|
||||
function = producer.function
|
||||
})
|
||||
}
|
||||
|
||||
for ((_, currentProducer) in currentProducers) {
|
||||
delete(currentProducer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package moe.odango.index.sync
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
abstract class ScheduledSync(val amount: Long, val timeUnit: TimeUnit) {
|
||||
abstract suspend fun run()
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
interface EntityHelper<T> {
|
||||
operator fun invoke(block: T.() -> Unit): T;
|
||||
}
|
||||
|
||||
@ -0,0 +1,18 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
import org.joda.time.DateTime
|
||||
import org.joda.time.format.ISODateTimeFormat
|
||||
|
||||
class ISOTextConverter : io.requery.Converter<DateTime, String> {
|
||||
override fun convertToMapped(type: Class<out DateTime>?, value: String?): DateTime? {
|
||||
return DateTime.parse(value ?: return null)
|
||||
}
|
||||
|
||||
override fun getPersistedType(): Class<String> = String::class.java
|
||||
override fun getMappedType(): Class<DateTime> = DateTime::class.java
|
||||
|
||||
override fun convertToPersisted(value: DateTime): String =
|
||||
value.toString(ISODateTimeFormat.dateTime())
|
||||
|
||||
override fun getPersistedSize(): Int? = null
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
enum class InfoSource {
|
||||
UserDefined,
|
||||
MyAnimeList,
|
||||
AniDB;
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
import java.sql.Date
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
|
||||
data class IntDate(val year: Int, val month: Int, val day: Int) {
|
||||
fun toDate(): Date = Date(year - 1900, month, day)
|
||||
|
||||
companion object {
|
||||
fun parse(format: String, input: String, locale: Locale = Locale.getDefault(Locale.Category.FORMAT)): IntDate? {
|
||||
val formatter = SimpleDateFormat(format, locale)
|
||||
|
||||
return try {
|
||||
formatter.parse(input)?.let {
|
||||
val nr = GregorianCalendar().apply {
|
||||
time = it
|
||||
}
|
||||
|
||||
IntDate(nr.get(Calendar.YEAR), nr.get(Calendar.MONTH), nr.get(Calendar.DAY_OF_MONTH))
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun Date.toIntDate() = IntDate(this.year + 1900, this.month, this.date)
|
||||
@ -0,0 +1,50 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
class MergeMap<T> {
|
||||
private var bucketIndex = 0L
|
||||
private val buckets: MutableMap<Long, MutableSet<T>> = mutableMapOf()
|
||||
private val bucketIndexByItem: MutableMap<T, Long> = mutableMapOf()
|
||||
|
||||
fun add(a: T, b: T) {
|
||||
val bucketA = bucketIndexByItem[a]
|
||||
val bucketB = bucketIndexByItem[b]
|
||||
if (bucketA == null && bucketB == null) {
|
||||
bucketIndexByItem[a] = bucketIndex
|
||||
bucketIndexByItem[b] = bucketIndex
|
||||
buckets[bucketIndex] = mutableSetOf(a, b)
|
||||
bucketIndex++
|
||||
return
|
||||
}
|
||||
|
||||
if (bucketB == bucketA) {
|
||||
return
|
||||
}
|
||||
|
||||
if (bucketA == null && bucketB != null) {
|
||||
bucketIndexByItem[a] = bucketB
|
||||
buckets.getOrPut(bucketB, ::mutableSetOf).add(a)
|
||||
return
|
||||
}
|
||||
|
||||
if (bucketB == null && bucketA != null) {
|
||||
bucketIndexByItem[b] = bucketA
|
||||
buckets.getOrPut(bucketA, ::mutableSetOf).add(b)
|
||||
return
|
||||
}
|
||||
|
||||
// Always false :)
|
||||
if (bucketA == null || bucketB == null) return
|
||||
|
||||
val bucket = buckets.getOrPut(bucketB, ::mutableSetOf)
|
||||
buckets.getOrPut(bucketA, ::mutableSetOf).addAll(bucket)
|
||||
for (item in bucket) {
|
||||
bucketIndexByItem[item] = bucketA
|
||||
}
|
||||
}
|
||||
|
||||
operator fun get(item: T): Set<T>? {
|
||||
return buckets[bucketIndexByItem[item] ?: return null]
|
||||
}
|
||||
|
||||
operator fun iterator(): Iterator<Set<T>> = buckets.values.iterator()
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
import io.inbot.eskotlinwrapper.MapBackedProperties
|
||||
import io.inbot.eskotlinwrapper.dsl.ESQuery
|
||||
|
||||
class NestedQuery : ESQuery(name = "nested") {
|
||||
var path: String by queryDetails.property()
|
||||
var ignoreUnmapped: Boolean by queryDetails.property()
|
||||
var innerHits: Map<Any, Any?> by queryDetails.property()
|
||||
var scoreMode: ScoreMode by queryDetails.property()
|
||||
var query: ESQuery by queryDetails.esQueryProperty()
|
||||
}
|
||||
|
||||
fun nested(path: String, config: NestedQuery.() -> Unit): NestedQuery {
|
||||
val q = NestedQuery()
|
||||
q.path = path
|
||||
config(q)
|
||||
return q
|
||||
}
|
||||
|
||||
@Suppress("EnumEntryName")
|
||||
enum class ScoreMode {
|
||||
avg,
|
||||
max,
|
||||
min,
|
||||
none,
|
||||
sum
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
enum class ProducerFunction {
|
||||
Studio,
|
||||
Producer,
|
||||
Licensor;
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
import io.inbot.eskotlinwrapper.dsl.ESQuery
|
||||
|
||||
class TermsSetQuery : ESQuery(name = "terms_set") {
|
||||
operator fun set(field: String, value: TermSetConfig) {
|
||||
queryDetails[field] = value.toMap()
|
||||
}
|
||||
|
||||
operator fun String.invoke(block: TermSetConfig.() -> Unit) {
|
||||
val config = TermSetConfig()
|
||||
block(config)
|
||||
set(this, config)
|
||||
}
|
||||
}
|
||||
|
||||
fun termsSet(block: TermsSetQuery.() -> Unit): TermsSetQuery {
|
||||
val q = TermsSetQuery()
|
||||
block(q)
|
||||
return q
|
||||
}
|
||||
|
||||
class TermSetConfig {
|
||||
private val map = mutableMapOf<String, Any>()
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
var terms: Set<Any>
|
||||
set(value) {
|
||||
map["terms"] = value as Any
|
||||
}
|
||||
get() = map["terms"] as? Set<Any> ?: emptySet()
|
||||
|
||||
var minimumShouldMatch: Int?
|
||||
get() = map["minimum_should_match_field"] as? Int
|
||||
set(value) { map["minimum_should_match_field"] = value as Any }
|
||||
|
||||
fun toMap() = map
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
import com.fasterxml.aalto.AsyncByteArrayFeeder
|
||||
import com.fasterxml.aalto.AsyncXMLStreamReader
|
||||
import com.fasterxml.aalto.stax.InputFactoryImpl
|
||||
import java.io.OutputStream
|
||||
|
||||
class XMLOutputStreamReader(private val afterWrite: AsyncXMLStreamReader<*>.() -> Unit) : OutputStream() {
|
||||
private val reader = InputFactoryImpl().createAsyncForByteArray()!!
|
||||
private val feeder = reader.inputFeeder!!
|
||||
|
||||
override fun write(b: ByteArray) {
|
||||
write(b, 0, b.size)
|
||||
}
|
||||
|
||||
override fun write(b: ByteArray, off: Int, len: Int) {
|
||||
feeder.feedInput(b, off, len)
|
||||
while (reader.next() != AsyncXMLStreamReader.EVENT_INCOMPLETE) {
|
||||
afterWrite(reader)
|
||||
}
|
||||
}
|
||||
|
||||
override fun write(b: Int) {
|
||||
write(byteArrayOf(b.toByte()), 0, 1)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
inline fun<reified T> helper(crossinline constructor: () -> T, crossinline init: T.() -> Unit = {}): EntityHelper<T> {
|
||||
return object : EntityHelper<T> {
|
||||
override fun invoke(block: T.() -> Unit): T {
|
||||
val entity = constructor()
|
||||
init(entity)
|
||||
block(entity)
|
||||
return entity
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,74 @@
|
||||
package moe.odango.index.utils
|
||||
|
||||
import org.jsoup.internal.StringUtil
|
||||
import org.jsoup.nodes.CDataNode
|
||||
import org.jsoup.nodes.Element
|
||||
import org.jsoup.nodes.Node
|
||||
import org.jsoup.nodes.TextNode
|
||||
import org.jsoup.select.NodeTraversor
|
||||
import org.jsoup.select.NodeVisitor
|
||||
|
||||
fun Element.brText(): String {
|
||||
val accum = StringUtil.borrowBuilder();
|
||||
NodeTraversor.traverse(object : NodeVisitor {
|
||||
override fun head(node: Node, depth: Int) {
|
||||
if (node is TextNode) {
|
||||
appendNormalisedText(accum, node);
|
||||
} else if (node is Element) {
|
||||
if (accum.isNotEmpty() &&
|
||||
((node.isBlock && !accum.lastIsWhitespace()) || node.tagName() == "br")
|
||||
) {
|
||||
if (node.tagName() == "br") {
|
||||
var lastIndex = accum.lastIndex
|
||||
while (accum[lastIndex] == ' ') {
|
||||
lastIndex--
|
||||
}
|
||||
accum.delete(lastIndex + 1, accum.length)
|
||||
accum.append('\n')
|
||||
} else {
|
||||
accum.append(' ');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun tail(node: Node, depth: Int) {
|
||||
// make sure there is a space between block tags and immediately following text nodes <div>One</div>Two should be "One Two".
|
||||
if (node is Element) {
|
||||
if (node.isBlock && (node.nextSibling() is TextNode) && (accum.lastIsWhitespace()))
|
||||
accum.append(' ');
|
||||
}
|
||||
|
||||
}
|
||||
}, this);
|
||||
|
||||
return StringUtil.releaseBuilder(accum).trim()
|
||||
}
|
||||
|
||||
fun StringBuilder.lastIsWhitespace() = lastOrNull() == ' ' || lastOrNull() == '\n'
|
||||
|
||||
fun appendNormalisedText(accum: StringBuilder, textNode: TextNode) {
|
||||
val text = textNode.wholeText;
|
||||
|
||||
if (preserveWhitespace(textNode.parentNode()) || textNode is CDataNode)
|
||||
accum.append(text);
|
||||
else
|
||||
StringUtil.appendNormalisedWhitespace(accum, text, accum.lastIsWhitespace());
|
||||
}
|
||||
|
||||
fun preserveWhitespace(node: Node): Boolean {
|
||||
// looks only at this element and five levels up, to prevent recursion & needless stack searches
|
||||
if (node is Element) {
|
||||
var el: Node? = node
|
||||
var i = 0;
|
||||
do {
|
||||
val ele = el ?: return false
|
||||
|
||||
if (ele is Element && ele.tag().preserveWhitespace())
|
||||
return true;
|
||||
el = ele.parent()
|
||||
i++;
|
||||
} while (i < 6 && el != null);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
HEELO WORLD
|
||||
@ -0,0 +1,208 @@
|
||||
package moe.odango.index.test.scraper.mal
|
||||
|
||||
import io.kotest.core.spec.style.StringSpec
|
||||
import io.kotest.matchers.collections.shouldHaveSize
|
||||
import io.kotest.matchers.shouldBe
|
||||
import io.kotest.matchers.shouldNotBe
|
||||
import moe.odango.index.entity.AnimeRelation
|
||||
import moe.odango.index.entity.MyAnimeListInfo
|
||||
import moe.odango.index.scraper.mal.AnimePageScraper
|
||||
import moe.odango.index.scraper.mal.AnimePageScraper.Genre
|
||||
import moe.odango.index.utils.IntDate
|
||||
import moe.odango.index.utils.ProducerFunction
|
||||
import java.time.Duration
|
||||
|
||||
class AnimePageScraperTest : StringSpec({
|
||||
"test-bodyscraper" {
|
||||
val body = this::class.java.getResourceAsStream("/test-pages/25313.html")
|
||||
.readAllBytes()
|
||||
.toString(Charsets.UTF_8)
|
||||
|
||||
val scraper = AnimePageScraper(body)
|
||||
|
||||
scraper.getDescription().trim() shouldBe """
|
||||
Bundled with limited edition of the 58th Gintama manga volume.
|
||||
|
||||
The tagline for the bundled anime reads, "It's time for all the Yorozuya members ...to wake up just one more time."
|
||||
|
||||
(Source: MAL News, edited)
|
||||
""".trim()
|
||||
|
||||
scraper.getImage() shouldBe "https://cdn.myanimelist.net/images/anime/12/64865.jpg"
|
||||
|
||||
val related = scraper.getRelated()
|
||||
|
||||
related shouldHaveSize 1
|
||||
related[0].id shouldBe 15417L
|
||||
related[0].type shouldBe AnimeRelation.RelationType.ParentStory
|
||||
|
||||
scraper.getPremiered() shouldBe null
|
||||
scraper.getAired() shouldNotBe null
|
||||
scraper.getAired()?.start shouldBe IntDate(2015, 3, 3)
|
||||
scraper.getSource() shouldBe "Manga"
|
||||
scraper.getDuration() shouldBe Duration.ofMinutes(24)
|
||||
scraper.getRating() shouldBe MyAnimeListInfo.Rating.PG13
|
||||
}
|
||||
|
||||
val items = mapOf(
|
||||
"616.html" to AnimePageScraper.Info(
|
||||
"Nurse Angel Ririka SOS",
|
||||
null,
|
||||
"ナースエンジェルりりかＳＯＳ",
|
||||
listOf(),
|
||||
"""
|
||||
The Evil Forces of Dark Joker are closing in on our planet after having destroyed the beautiful planet of Queen Earth. Now, 10 year old Moriya Ririka, with the help of her childhood friend Seiya and the mysterious Kanon, must transform into the Nurse Angel and find the elusive Flower of Life, the only way to defeat the evil forces. The Flower of Life, that once bloomed all over the Earth, is where no one thought it ever would be. And Ririka must make the hardest decision of her life in order to acquire it and rid the universe of evil once and for all.
|
||||
|
||||
(Source: ANN)
|
||||
""".trimIndent(),
|
||||
MyAnimeListInfo.ReleaseType.TV,
|
||||
35,
|
||||
"Manga",
|
||||
"https://cdn.myanimelist.net/images/anime/10/10506.jpg",
|
||||
listOf(
|
||||
Genre.DRAMA,
|
||||
Genre.FANTASY,
|
||||
Genre.MAGIC,
|
||||
Genre.SHOUJO
|
||||
),
|
||||
AnimePageScraper.Aired(
|
||||
IntDate(1995, 6, 7),
|
||||
IntDate(1996, 2, 29)
|
||||
),
|
||||
AnimePageScraper.Premiered(AnimePageScraper.Premiered.Season.Summer, 1995),
|
||||
MyAnimeListInfo.Rating.PG,
|
||||
Duration.ofMinutes(24),
|
||||
listOf(),
|
||||
listOf(
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Producer, AnimePageScraper.Producer(16, "TV Tokyo")),
|
||||
AnimePageScraper.ProducerRelation(
|
||||
ProducerFunction.Producer,
|
||||
AnimePageScraper.Producer(139, "Nihon Ad Systems")
|
||||
),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Studio, AnimePageScraper.Producer(36, "Gallop"))
|
||||
)
|
||||
),
|
||||
"817.html" to AnimePageScraper.Info(
|
||||
"Tactical Roar",
|
||||
null,
|
||||
"タクティカルロア",
|
||||
listOf("Tactical Rawr"),
|
||||
"""
|
||||
In the near future the world's climate shifted creating in the Western Pacific a perpetual super cyclone: the Grand Roar that altered the earth, flooding most countries. Shipping and navigation became important to nations and following the appearance of ocean pirates, necessisated companies to hire escort cruisers to safeguard their investments. Hyousuke Nagimiya is a system engineer that was comissioned to upgrade the Pascal Magi manned by an entire crew of women with its captain, Misaki Nanaha. Together the crew strives to prove themselves to their detractors that they are no mere 'Alice Brand'. Yet as they go about their mission a larger global conspiracy seems to be working behind the scenes to take advantage of this new world order.
|
||||
|
||||
(Source: ANN)
|
||||
""".trimIndent(),
|
||||
MyAnimeListInfo.ReleaseType.TV,
|
||||
13,
|
||||
"Unknown",
|
||||
"https://cdn.myanimelist.net/images/anime/8/61829.jpg",
|
||||
listOf(
|
||||
Genre.COMEDY,
|
||||
Genre.MILITARY,
|
||||
Genre.ROMANCE,
|
||||
Genre.SCIFI
|
||||
),
|
||||
AnimePageScraper.Aired(
|
||||
IntDate(2006, 0, 8),
|
||||
IntDate(2006, 3, 2)
|
||||
),
|
||||
AnimePageScraper.Premiered(AnimePageScraper.Premiered.Season.Winter, 2006),
|
||||
MyAnimeListInfo.Rating.PG13,
|
||||
Duration.ofMinutes(25),
|
||||
listOf(
|
||||
AnimePageScraper.Relation(AnimeRelation.RelationType.SideStory, 1790)
|
||||
),
|
||||
listOf(
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Producer, AnimePageScraper.Producer(23, "Bandai Visual")),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Producer, AnimePageScraper.Producer(104, "Lantis")),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Studio, AnimePageScraper.Producer(60, "Actas"))
|
||||
)
|
||||
),
|
||||
"1558.html" to AnimePageScraper.Info(
|
||||
"Yarima Queen",
|
||||
"Sex Demon Queen",
|
||||
"ヤーリマクィーン",
|
||||
listOf(),
|
||||
"""
|
||||
The sorceress Kuri uses her magic to defend herself from perverted monsters and demons, but her partner, Rima, would much rather do perverted things than defend herself. When the two save a woman from a gang rape, they catch the eye of an evil Sex Queen and her dog-demons. When the sex demons release the passions of Kuri and Rima, even the duo`s formidable powers will be useless. Don`t miss it as all four girls redefine the meaning of Doggie Style!
|
||||
|
||||
(Source: AniDB)
|
||||
""".trimIndent(),
|
||||
MyAnimeListInfo.ReleaseType.OVA,
|
||||
1,
|
||||
"Unknown",
|
||||
"https://cdn.myanimelist.net/images/anime/10/41571.jpg",
|
||||
listOf(
|
||||
Genre.COMEDY,
|
||||
Genre.DEMONS,
|
||||
Genre.FANTASY,
|
||||
Genre.HENTAI,
|
||||
Genre.MAGIC,
|
||||
Genre.PARODY
|
||||
),
|
||||
AnimePageScraper.Aired(
|
||||
IntDate(2000, 5, 25)
|
||||
),
|
||||
null,
|
||||
MyAnimeListInfo.Rating.Rx,
|
||||
Duration.ofMinutes(30),
|
||||
listOf(),
|
||||
listOf(
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Producer, AnimePageScraper.Producer(48, "AIC")),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Producer, AnimePageScraper.Producer(152, "Green Bunny")),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Licensor, AnimePageScraper.Producer(250, "Media Blasters")),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Licensor, AnimePageScraper.Producer(595, "NYAV Post"))
|
||||
)
|
||||
),
|
||||
"1581.html" to AnimePageScraper.Info(
|
||||
"Gift: Eternal Rainbow",
|
||||
null,
|
||||
"ギフト～ eternal rainbow",
|
||||
listOf(),
|
||||
"""
|
||||
Amaumi Haruhiko is a high school student who attends Shimano Academy in a town called Narasakicho. Narasakicho contains an unknown rainbow which constantly overlooks the town and is related to granting a magical wish called "Gift." Gift is a once-in-a-lifetime present between two people.
|
||||
|
||||
As a child, Haruhiko has been close with his childhood friend, Kirino, until he obtains a new non-blood sister by the name of Riko. Haruhiko develops a strong relationship with Riko until they sadly depart due to the fact Haruhiko's father could no longer support the two of them.
|
||||
|
||||
After some times passes by, Riko finally returns to the town of Narasakicho, and along with Kirino, starts to attend Shimano Academy with Haruhiko. The series revolves around the relationship among these main protagonists and slowly reveals the story behind both Gift and the rainbow.
|
||||
""".trimIndent(),
|
||||
MyAnimeListInfo.ReleaseType.TV,
|
||||
12,
|
||||
"Visual novel",
|
||||
"https://cdn.myanimelist.net/images/anime/2/75540.jpg",
|
||||
listOf(
|
||||
Genre.COMEDY,
|
||||
Genre.DRAMA,
|
||||
Genre.HAREM,
|
||||
Genre.MAGIC,
|
||||
Genre.ROMANCE,
|
||||
Genre.SCHOOL
|
||||
),
|
||||
AnimePageScraper.Aired(
|
||||
IntDate(2006, 9, 6),
|
||||
IntDate(2006, 11, 22)
|
||||
),
|
||||
AnimePageScraper.Premiered(AnimePageScraper.Premiered.Season.Fall, 2006),
|
||||
MyAnimeListInfo.Rating.PG13,
|
||||
Duration.ofMinutes(24),
|
||||
listOf(
|
||||
AnimePageScraper.Relation(AnimeRelation.RelationType.SideStory, 2784)
|
||||
),
|
||||
listOf(
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Producer, AnimePageScraper.Producer(829, "Studio Jack")),
|
||||
AnimePageScraper.ProducerRelation(ProducerFunction.Studio, AnimePageScraper.Producer(28, "OLM"))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
for ((file, info) in items) {
|
||||
"Test $file" {
|
||||
val html = this::class.java.getResourceAsStream("/test-pages/$file")
|
||||
.readAllBytes()
|
||||
.toString(Charsets.UTF_8)
|
||||
|
||||
AnimePageScraper(html)
|
||||
.getInfo() shouldBe info
|
||||
}
|
||||
}
|
||||
})
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue