Browse Source

Initial commit

🏳️‍⚧️
eater 1 year ago
commit
d62969aafc
Signed by: eater GPG Key ID: AD2560A0F84F0759
  1. 6
      .gitignore
  2. 3
      README.md
  3. 80
      build.gradle
  4. 8
      config/test.toml
  5. 9
      docker-compose.yml
  6. 1
      gradle.properties
  7. BIN
      gradle/wrapper/gradle-wrapper.jar
  8. 6
      gradle/wrapper/gradle-wrapper.properties
  9. 183
      gradlew
  10. 100
      gradlew.bat
  11. 2
      settings.gradle
  12. 26
      src/main/kotlin/moe/odango/index/cli/AniDBSync.kt
  13. 28
      src/main/kotlin/moe/odango/index/cli/DatabaseMigration.kt
  14. 10
      src/main/kotlin/moe/odango/index/cli/ElasticIndex.kt
  15. 10
      src/main/kotlin/moe/odango/index/cli/HTTPServer.kt
  16. 13
      src/main/kotlin/moe/odango/index/cli/MyAnimeListListingSync.kt
  17. 13
      src/main/kotlin/moe/odango/index/cli/MyAnimeListPageSync.kt
  18. 8
      src/main/kotlin/moe/odango/index/config/DatabaseConfiguration.kt
  19. 8
      src/main/kotlin/moe/odango/index/config/ElasticSearchConfiguration.kt
  20. 6
      src/main/kotlin/moe/odango/index/config/IndexConfiguration.kt
  21. 59
      src/main/kotlin/moe/odango/index/container.kt
  22. 58
      src/main/kotlin/moe/odango/index/entity/AniDBInfo.kt
  23. 50
      src/main/kotlin/moe/odango/index/entity/Anime.kt
  24. 27
      src/main/kotlin/moe/odango/index/entity/AnimeGenre.kt
  25. 28
      src/main/kotlin/moe/odango/index/entity/AnimeProducer.kt
  26. 82
      src/main/kotlin/moe/odango/index/entity/AnimeRelation.kt
  27. 20
      src/main/kotlin/moe/odango/index/entity/AnimeSeries.kt
  28. 27
      src/main/kotlin/moe/odango/index/entity/AnimeTag.kt
  29. 21
      src/main/kotlin/moe/odango/index/entity/Genre.kt
  30. 89
      src/main/kotlin/moe/odango/index/entity/MyAnimeListInfo.kt
  31. 22
      src/main/kotlin/moe/odango/index/entity/Producer.kt
  32. 25
      src/main/kotlin/moe/odango/index/entity/Tag.kt
  33. 63
      src/main/kotlin/moe/odango/index/entity/Title.kt
  34. 141
      src/main/kotlin/moe/odango/index/es/Indexer.kt
  35. 13
      src/main/kotlin/moe/odango/index/es/dto/AnimeDTO.kt
  36. 8
      src/main/kotlin/moe/odango/index/es/dto/AnimeDescriptionDTO.kt
  37. 10
      src/main/kotlin/moe/odango/index/es/dto/AnimeTitleDTO.kt
  38. 41
      src/main/kotlin/moe/odango/index/http/Server.kt
  39. 100
      src/main/kotlin/moe/odango/index/http/graphql/AnimeService.kt
  40. 7
      src/main/kotlin/moe/odango/index/http/graphql/dto/AnimeItem.kt
  41. 11
      src/main/kotlin/moe/odango/index/http/graphql/dto/AnimeTitleItem.kt
  42. 6
      src/main/kotlin/moe/odango/index/http/graphql/dto/AutoCompleteItem.kt
  43. 30
      src/main/kotlin/moe/odango/index/main.kt
  44. 33
      src/main/kotlin/moe/odango/index/scraper/mal/AnimeListScraper.kt
  45. 246
      src/main/kotlin/moe/odango/index/scraper/mal/AnimePageScraper.kt
  46. 178
      src/main/kotlin/moe/odango/index/sync/AniDBTitleSync.kt
  47. 318
      src/main/kotlin/moe/odango/index/sync/AniDBXMLSync.kt
  48. 122
      src/main/kotlin/moe/odango/index/sync/MyAnimeListListingSync.kt
  49. 327
      src/main/kotlin/moe/odango/index/sync/MyAnimeListPageSync.kt
  50. 7
      src/main/kotlin/moe/odango/index/sync/ScheduledSync.kt
  51. 6
      src/main/kotlin/moe/odango/index/utils/EntityHelper.kt
  52. 18
      src/main/kotlin/moe/odango/index/utils/ISOTextConverter.kt
  53. 7
      src/main/kotlin/moe/odango/index/utils/InfoSource.kt
  54. 29
      src/main/kotlin/moe/odango/index/utils/IntDate.kt
  55. 50
      src/main/kotlin/moe/odango/index/utils/MergeMap.kt
  56. 28
      src/main/kotlin/moe/odango/index/utils/NestedQuery.kt
  57. 7
      src/main/kotlin/moe/odango/index/utils/ProducerFunction.kt
  58. 38
      src/main/kotlin/moe/odango/index/utils/TermsSetQuery.kt
  59. 26
      src/main/kotlin/moe/odango/index/utils/XMLOutputStreamReader.kt
  60. 13
      src/main/kotlin/moe/odango/index/utils/helper.kt
  61. 74
      src/main/kotlin/moe/odango/index/utils/textWithBreaks.kt
  62. 1
      src/main/resources/web/index.html
  63. 208
      src/test/kotlin/moe/odango/index/test/scraper/mal/AnimePageScraperTest.kt
  64. 1288
      src/test/resources/test-pages/1558.html
  65. 1629
      src/test/resources/test-pages/1581.html
  66. 1213
      src/test/resources/test-pages/25313.html
  67. 1435
      src/test/resources/test-pages/616.html
  68. 1237
      src/test/resources/test-pages/817.html

6
.gitignore

@ -0,0 +1,6 @@
*.sqlite
*.sqlite
out
build
.gradle
.idea

3
README.md

@ -0,0 +1,3 @@
# Index (Librorum Animum)

80
build.gradle

@ -0,0 +1,80 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.3.72'
id 'org.jetbrains.kotlin.kapt' version '1.3.72'
id 'application'
}
group 'moe.odango'
version '1.0-SNAPSHOT'
repositories {
mavenCentral()
jcenter()
maven { url 'https://jitpack.io' }
}
test {
useJUnitPlatform()
}
dependencies {
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
implementation 'org.xerial:sqlite-jdbc:3.31.1'
implementation 'io.ktor:ktor-server-core:1.3.2'
implementation 'io.ktor:ktor-server-netty:1.3.2'
implementation 'io.ktor:ktor-jackson:1.3.2'
implementation 'io.requery:requery:1.6.1'
implementation 'io.requery:requery-kotlin:1.6.1'
kapt 'io.requery:requery-processor:1.6.1'
implementation 'org.slf4j:slf4j-simple:1.6.1'
implementation 'org.apache.logging.log4j:log4j-core:2.13.3'
implementation 'com.github.kittinunf.fuel:fuel:2.2.0'
implementation 'com.github.kittinunf.fuel:fuel-coroutines:2.2.0'
implementation 'com.fasterxml:aalto-xml:1.2.1'
implementation 'org.kodein.di:kodein-di:7.0.0'
implementation 'com.github.ajalt:clikt:2.7.1'
implementation 'org.xerial:sqlite-jdbc:3.8.11.2'
implementation 'org.postgresql:postgresql:42.2.5'
implementation 'com.moandjiezana.toml:toml4j:0.7.2'
implementation 'org.apache.commons:commons-compress:1.18'
implementation 'com.expediagroup:graphql-kotlin-schema-generator:3.1.1'
implementation 'com.graphql-java:graphql-java:15.0'
implementation 'com.github.excitement-engineer:ktor-graphql:1.0.0'
implementation 'org.jsoup:jsoup:1.13.1'
implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.7.0'
implementation 'com.github.jillesvangurp:es-kotlin-wrapper-client:1.0-X-Beta-5-7.7.0'
testImplementation 'io.kotest:kotest-runner-junit5-jvm:4.0.5'
testImplementation 'io.kotest:kotest-assertions-core-jvm:4.0.5'
testImplementation 'io.kotest:kotest-property-jvm:4.0.5'
}
application {
mainClassName = 'moe.odango.index.MainKt'
}
compileKotlin {
kotlinOptions.jvmTarget = '1.8'
}
compileTestKotlin {
kotlinOptions.jvmTarget = '1.8'
}

8
config/test.toml

@ -0,0 +1,8 @@
[database]
driver = "sqlite"
uri = "jdbc:sqlite:test.sql"
[elastic]
index = "anime"
replicas = 0
shards = 2

9
docker-compose.yml

@ -0,0 +1,9 @@
version: '3.7'
services:
es:
image: elasticsearch:7.7.0
environment:
discovery.type: single-node
ports:
- 9200:9200
- 9300:9300

1
gradle.properties

@ -0,0 +1 @@
kotlin.code.style=official

BIN
gradle/wrapper/gradle-wrapper.jar

6
gradle/wrapper/gradle-wrapper.properties

@ -0,0 +1,6 @@
#Sat Jun 06 16:52:37 CEST 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-6.1-all.zip
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStorePath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME

183
gradlew

@ -0,0 +1,183 @@
#!/usr/bin/env sh
#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn () {
echo "$*"
}
die () {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
NONSTOP* )
nonstop=true
;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin or MSYS, switch paths to Windows format before running java
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=`expr $i + 1`
done
case $i in
0) set -- ;;
1) set -- "$args0" ;;
2) set -- "$args0" "$args1" ;;
3) set -- "$args0" "$args1" "$args2" ;;
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Escape application args
save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=`save "$@"`
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
exec "$JAVACMD" "$@"

100
gradlew.bat

@ -0,0 +1,100 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windows variants
if not "%OS%" == "Windows_NT" goto win9xME_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

2
settings.gradle

@ -0,0 +1,2 @@
rootProject.name = 'index'

26
src/main/kotlin/moe/odango/index/cli/AniDBSync.kt

@ -0,0 +1,26 @@
package moe.odango.index.cli
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.options.option
import kotlinx.coroutines.runBlocking
import moe.odango.index.sync.AniDBTitleSync
import java.io.File
class AniDBSync : CliktCommand(name = "anidb",help = "Sync the titles from AniDB with the local database") {
private val file by option(help = "Read from file instead of downloading")
override fun run() {
val sync = AniDBTitleSync()
val file = file
runBlocking {
if (file == null) {
sync.run()
} else {
val fileObj = File(file).takeIf { it.isFile } ?: throw RuntimeException("File doesn't exist")
sync.syncWithFile(fileObj)
}
}
}
}

28
src/main/kotlin/moe/odango/index/cli/DatabaseMigration.kt

@ -0,0 +1,28 @@
package moe.odango.index.cli
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
import io.requery.sql.KotlinConfiguration
import io.requery.sql.SchemaModifier
import io.requery.sql.TableCreationMode
import moe.odango.index.di
import org.kodein.di.instance
class DatabaseMigration : CliktCommand(name = "db:create") {
private val print by option().flag(default = false)
private val exec by option().flag(default = false)
override fun run() {
val config by di.instance<KotlinConfiguration>()
val modifier = SchemaModifier(config)
if (exec) {
modifier.createTables(TableCreationMode.CREATE_NOT_EXISTS)
}
if (print) {
println(modifier.createTablesString(TableCreationMode.CREATE_NOT_EXISTS))
}
}
}

10
src/main/kotlin/moe/odango/index/cli/ElasticIndex.kt

@ -0,0 +1,10 @@
package moe.odango.index.cli
import com.github.ajalt.clikt.core.CliktCommand
import moe.odango.index.es.Indexer
class ElasticIndex : CliktCommand(name = "elastic:index") {
override fun run() {
Indexer().run()
}
}

10
src/main/kotlin/moe/odango/index/cli/HTTPServer.kt

@ -0,0 +1,10 @@
package moe.odango.index.cli
import com.github.ajalt.clikt.core.CliktCommand
import moe.odango.index.http.Server
class HTTPServer : CliktCommand(name = "http:serve") {
override fun run() {
Server().run()
}
}

13
src/main/kotlin/moe/odango/index/cli/MyAnimeListListingSync.kt

@ -0,0 +1,13 @@
package moe.odango.index.cli
import com.github.ajalt.clikt.core.CliktCommand
import kotlinx.coroutines.runBlocking
import moe.odango.index.sync.MyAnimeListListingSync
class MyAnimeListListingSync : CliktCommand(name = "mal:sync-listing") {
override fun run() {
runBlocking {
MyAnimeListListingSync().run()
}
}
}

13
src/main/kotlin/moe/odango/index/cli/MyAnimeListPageSync.kt

@ -0,0 +1,13 @@
package moe.odango.index.cli
import com.github.ajalt.clikt.core.CliktCommand
import kotlinx.coroutines.runBlocking
import moe.odango.index.sync.MyAnimeListPageSync
class MyAnimeListPageSync : CliktCommand(name = "mal:sync-page") {
override fun run() {
runBlocking {
MyAnimeListPageSync().run()
}
}
}

8
src/main/kotlin/moe/odango/index/config/DatabaseConfiguration.kt

@ -0,0 +1,8 @@
package moe.odango.index.config
data class DatabaseConfiguration(
val uri: String,
val driver: String = "sqlite",
val username: String? = null,
val password: String? = null
)

8
src/main/kotlin/moe/odango/index/config/ElasticSearchConfiguration.kt

@ -0,0 +1,8 @@
package moe.odango.index.config
data class ElasticSearchConfiguration(
val host: String = "localhost:9200",
val index: String = "index_anime",
val replicas: Int = 0,
val shards: Int = 2
)

6
src/main/kotlin/moe/odango/index/config/IndexConfiguration.kt

@ -0,0 +1,6 @@
package moe.odango.index.config
data class IndexConfiguration(
val database: DatabaseConfiguration = DatabaseConfiguration("jdbc:sqlite:index.sqlite"),
val elastic: ElasticSearchConfiguration = ElasticSearchConfiguration()
)

59
src/main/kotlin/moe/odango/index/container.kt

@ -0,0 +1,59 @@
package moe.odango.index
import com.moandjiezana.toml.Toml
import io.inbot.eskotlinwrapper.IndexRepository
import io.requery.Persistable
import io.requery.sql.KotlinConfiguration
import io.requery.sql.KotlinEntityDataStore
import moe.odango.index.config.IndexConfiguration
import moe.odango.index.entity.Models
import moe.odango.index.es.dto.AnimeDTO
import org.apache.http.HttpHost
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.client.indexRepository
import org.kodein.di.*
import org.sqlite.SQLiteDataSource
import java.io.File
import javax.sql.CommonDataSource
val di = DI {
bind<RestHighLevelClient>() with provider {
RestHighLevelClient(RestClient.builder(HttpHost.create(instance<IndexConfiguration>().elastic.host)))
}
bind<IndexRepository<AnimeDTO>>() with provider {
instance<RestHighLevelClient>().indexRepository<AnimeDTO>(instance<IndexConfiguration>().elastic.index)
}
bind<KotlinConfiguration>() with singleton { KotlinConfiguration(Models.DEFAULT, dataSource = instance()) }
bind<KotlinEntityDataStore<Persistable>>() with singleton { KotlinEntityDataStore<Persistable>(instance()) }
constant(tag = "config-file") with mutableListOf(
"/etc/index/config.toml",
System.getenv("HOME") + "/.index/config.toml"
)
bind<IndexConfiguration>() with singleton {
val toml = instance<MutableList<String>>("config-file")
.find { File(it).isFile }
?.let { File(it).readText(Charsets.UTF_8) }
toml?.let {
Toml()
.read(toml)
.to(IndexConfiguration::class.java)
} ?: IndexConfiguration()
}
bind<CommonDataSource>() with singleton {
val config: IndexConfiguration = instance()
when (config.database.driver) {
"sqlite" -> SQLiteDataSource().apply {
url = config.database.uri
}
else -> throw RuntimeException("Database driver '${config.database.driver}' not recognized")
}
}
}

58
src/main/kotlin/moe/odango/index/entity/AniDBInfo.kt

@ -0,0 +1,58 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface AniDBInfo : Persistable {
@get:Key
val id: UUID
@get:OneToOne
@get:ForeignKey
val anime: Anime
var type: ReleaseType
var restricted: Boolean
var episodes: Int?
var description: String
var image: String?
var startYear: Int?
var startMonth: Int?
var startDay: Int?
var endYear: Int?
var endMonth: Int?
var endDay: Int?
enum class ReleaseType {
Movie,
MusicVideo,
OVA,
Other,
TVSeries,
TVSpecial,
Web,
Unknown;
companion object {
fun fromAniDBString(value: String): ReleaseType = when (value.toLowerCase()) {
"movie" -> Movie
"music video" -> MusicVideo
"ova" -> OVA
"other" -> Other
"tv series" -> TVSeries
"tv special" -> TVSpecial
"web" -> Web
else -> Unknown
}
}
}
companion object : EntityHelper<AniDBInfoEntity> by helper(::AniDBInfoEntity, {
setId(UUID.randomUUID())
})
}

50
src/main/kotlin/moe/odango/index/entity/Anime.kt

@ -0,0 +1,50 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface Anime : Persistable {
@get:Key
val id: UUID
@get:Index
var aniDbId: Long?
@get:OneToOne
val aniDbInfo: AniDBInfo?
@get:OneToMany(mappedBy = "anime")
val titles: List<Title>
@get:Index
var myAnimeListId: Long?
@get:ManyToMany
@get:JunctionTable(type = AnimeProducer::class)
val producers: List<Producer>
@get:OneToOne
val myAnimeListInfo: MyAnimeListInfo?
@get:OneToMany(mappedBy = "anime_from")
val relatedTo: List<AnimeRelation>
@get:OneToMany(mappedBy = "anime_to")
val relatedFrom: List<AnimeRelation>
@get:OneToMany(mappedBy = "anime")
val genres: List<AnimeGenre>
@get:ManyToOne
@get:ForeignKey
var series: AnimeSeries?
val replacedWith: UUID?
companion object : EntityHelper<AnimeEntity> by helper(::AnimeEntity, {
setId(UUID.randomUUID())
})
}

27
src/main/kotlin/moe/odango/index/entity/AnimeGenre.kt

@ -0,0 +1,27 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.InfoSource
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface AnimeGenre : Persistable {
@get:Key
val id: UUID
@get:ManyToOne
@get:ForeignKey
val anime: Anime
@get:ManyToOne
@get:ForeignKey
val genre: Genre
val source: InfoSource
companion object : EntityHelper<AnimeGenreEntity> by helper(::AnimeGenreEntity, {
setId(UUID.randomUUID())
})
}

28
src/main/kotlin/moe/odango/index/entity/AnimeProducer.kt

@ -0,0 +1,28 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.InfoSource
import moe.odango.index.utils.ProducerFunction
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface AnimeProducer : Persistable {
@get:Key
val id: UUID
@get:ManyToOne
val anime: Anime
@get:ManyToOne
val producer: Producer
@get:Column(name = "producer_function")
var function: ProducerFunction
val source: InfoSource
companion object : EntityHelper<AnimeProducerEntity> by helper(::AnimeProducerEntity, {
setId(UUID.randomUUID())
})
}

82
src/main/kotlin/moe/odango/index/entity/AnimeRelation.kt

@ -0,0 +1,82 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.InfoSource
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface AnimeRelation : Persistable {
@get:Key
val id: UUID
@get:ManyToOne
@get:ForeignKey
@get:Column(name = "anime_from")
val from: Anime
@get:ManyToOne
@get:ForeignKey
@get:Column(name = "anime_to")
val to: Anime
var relation: RelationType
val source: InfoSource
companion object : EntityHelper<AnimeRelationEntity> by helper(::AnimeRelationEntity, {
setId(UUID.randomUUID())
})
enum class RelationType {
Prequel,
Sequel,
SideStory,
FullStory,
ParentStory,
Summary,
SpinOff,
AlternateVersion,
AlternateSetting,
SameSetting,
Other,
Character;
val inverse: RelationType
get() {
return when (this) {
Prequel -> Sequel
Sequel -> Prequel
Summary -> FullStory
FullStory -> Summary
SideStory -> ParentStory
SpinOff -> ParentStory
ParentStory -> SideStory
AlternateSetting -> AlternateSetting
AlternateVersion -> AlternateVersion
SameSetting -> SameSetting
Other -> Other
Character -> Character
}
}
companion object {
fun fromString(type: String): RelationType {
return when (type.toLowerCase()) {
"alternate setting" -> AlternateSetting
"alternate version" -> AlternateVersion
"prequel" -> Prequel
"sequel" -> Sequel
"spin off" -> SpinOff
"summary" -> Summary
"side story" -> SideStory
"full story" -> FullStory
"parent story" -> ParentStory
"character" -> Character
"same setting" -> SameSetting
else -> Other
}
}
}
}
}

20
src/main/kotlin/moe/odango/index/entity/AnimeSeries.kt

@ -0,0 +1,20 @@
package moe.odango.index.entity
import io.requery.Entity
import io.requery.Key
import io.requery.Persistable
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface AnimeSeries : Persistable {
@get:Key
val id: UUID
val name: String?
var replacedWith: UUID?
companion object : EntityHelper<AnimeSeriesEntity> by helper(::AnimeSeriesEntity, {
setId(UUID.randomUUID())
})
}

27
src/main/kotlin/moe/odango/index/entity/AnimeTag.kt

@ -0,0 +1,27 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.InfoSource
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface AnimeTag : Persistable {
@get:Key
val id: UUID
@get:ManyToOne
@get:ForeignKey
val anime: Anime
@get:ManyToOne
@get:ForeignKey
val tag: Tag
val source: InfoSource
val spoiler: Boolean
companion object : EntityHelper<AnimeTagEntity> by helper(::AnimeTagEntity, {
setId(UUID.randomUUID())
})
}

21
src/main/kotlin/moe/odango/index/entity/Genre.kt

@ -0,0 +1,21 @@
package moe.odango.index.entity
import io.requery.Entity
import io.requery.Key
import io.requery.Persistable
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.InfoSource
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface Genre : Persistable {
@get:Key
val id: UUID
val myAnimeListId: Int
val name: String
companion object : EntityHelper<GenreEntity> by helper(::GenreEntity, {
setId(UUID.randomUUID())
})
}

89
src/main/kotlin/moe/odango/index/entity/MyAnimeListInfo.kt

@ -0,0 +1,89 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.scraper.mal.AnimePageScraper
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.ISOTextConverter
import moe.odango.index.utils.helper
import moe.odango.index.utils.toIntDate
import org.joda.time.DateTime
import java.sql.Date
import java.util.*
@Entity
interface MyAnimeListInfo : Persistable {
@get:Key
val id: UUID
@get:OneToOne
@get:ForeignKey
val anime: Anime
var releaseType: ReleaseType
var episodes: Int?
@get:Convert(ISOTextConverter::class)
var lastScrape: DateTime?
var rating: Rating?
var image: String?
var description: String?
var source: String?
var airedStart: Date?
var airedEnd: Date?
var premieredSeason: String?
var premieredYear: Int?
var duration: Int?
@get:Transient
val aired: AnimePageScraper.Aired?
get() = airedStart?.let { AnimePageScraper.Aired(it.toIntDate(), airedEnd?.toIntDate()) }
@get:Transient
val premiered: AnimePageScraper.Premiered?
get() = premieredSeason
?.let(AnimePageScraper.Premiered.Season.Companion::fromString)
?.let { s ->
premieredYear?.let { y ->
AnimePageScraper.Premiered(s, y)
}
}
enum class Rating {
G,
PG,
PG13,
R,
RPlus,
Rx;
companion object {
fun fromString(rating: String): Rating? = when (rating) {
"G" -> G
"PG" -> PG
"PG-13" -> PG13
"R" -> R
"R+" -> RPlus
"Rx" -> Rx
else -> null
}
}
}
enum class ReleaseType {
TV,
ONA,
OVA,
Movie,
Special,
Music,
Unknown
}
companion object : EntityHelper<MyAnimeListInfoEntity> by helper(::MyAnimeListInfoEntity, {
setId(UUID.randomUUID())
episodes = null
releaseType = ReleaseType.Unknown
lastScrape = null
})
}

22
src/main/kotlin/moe/odango/index/entity/Producer.kt

@ -0,0 +1,22 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface Producer : Persistable {
@get:Key
val id: UUID
val myAnimeListId: Int?
val name: String
@get:ManyToMany
val animes: List<Anime>
companion object : EntityHelper<ProducerEntity> by helper(::ProducerEntity, {
setId(UUID.randomUUID())
})
}

25
src/main/kotlin/moe/odango/index/entity/Tag.kt

@ -0,0 +1,25 @@
package moe.odango.index.entity
import io.requery.Entity
import io.requery.Key
import io.requery.Persistable
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface Tag : Persistable {
@get:Key
val id: UUID
val aniDbId: Long?
val parentId: UUID
val name: String
val description: String
val spoiler: Boolean
companion object : EntityHelper<TagEntity> by helper(::TagEntity, {
setId(UUID.randomUUID())
})
}

63
src/main/kotlin/moe/odango/index/entity/Title.kt

@ -0,0 +1,63 @@
package moe.odango.index.entity
import io.requery.*
import moe.odango.index.utils.EntityHelper
import moe.odango.index.utils.InfoSource
import moe.odango.index.utils.helper
import java.util.*
@Entity
interface Title : Persistable {
@get:Key
val id: UUID
@get:Index
var name: String
@get:ManyToOne
@get:ForeignKey
val anime: Anime
val source: InfoSource
@get:Column(name = "lang")
val language: String?
val type: TitleType
var hidden: Boolean
enum class TitleType {
Synonym,
TitleCard,
Kana,
Short,
Official,
Main;
fun toAniDBString() = when (this) {
Main -> "main"
Synonym -> "syn"
TitleCard -> "card"
Kana -> "kana"
Short -> "short"
Official -> "official"
}
companion object {
fun fromAniDBString(type: String) = when (type) {
"card" -> TitleCard
"official" -> Official
"short" -> Short
"syn" -> Synonym
"kana" -> Kana
"main" -> Main
else -> Synonym
}
}
}
companion object : EntityHelper<TitleEntity> by helper(::TitleEntity, {
setId(UUID.randomUUID())
})
}

141
src/main/kotlin/moe/odango/index/es/Indexer.kt

@ -0,0 +1,141 @@
package moe.odango.index.es
import io.inbot.eskotlinwrapper.IndexRepository
import io.requery.Persistable
import io.requery.kotlin.invoke
import io.requery.sql.KotlinEntityDataStore
import moe.odango.index.config.IndexConfiguration
import moe.odango.index.di
import moe.odango.index.entity.Anime
import moe.odango.index.es.dto.AnimeDescriptionDTO
import moe.odango.index.es.dto.AnimeTitleDTO
import moe.odango.index.utils.InfoSource
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.client.configure
import org.elasticsearch.client.indices.GetIndexRequest
import org.kodein.di.instance
import moe.odango.index.es.dto.AnimeDTO as AnimeDTO
class Indexer {
private val indexRepo by di.instance<IndexRepository<AnimeDTO>>()
private val indexConfig by di.instance<IndexConfiguration>()
private val client by di.instance<RestHighLevelClient>()
private val config by lazy { indexConfig.elastic }
private val entityStore by di.instance<KotlinEntityDataStore<Persistable>>()
fun run() {
createIndex()
index()
client.close()
}
fun createIndex() {
val indexExists = client.indices()
.exists(GetIndexRequest(config.index), RequestOptions.DEFAULT);
if (indexExists)
return
indexRepo.createIndex {
configure {
settings {
replicas = config.replicas
shards = config.shards
addTokenizer("autocomplete") {
this["type"] = "edge_ngram"
this["min_gram"] = 2
this["max_gram"] = 10
this["token_chars"] = listOf("letter")
}
addAnalyzer("autocomplete") {
this["tokenizer"] = "autocomplete"
this["filter"] = listOf("lowercase")
}
addAnalyzer("autocomplete_search") {
this["tokenizer"] = "lowercase"
}
}
mappings {
nestedField("title") {
text("name") {
analyzer = "autocomplete"
searchAnalyzer = "autocomplete_search"
}
}
nestedField("description") {
text("text") {
analyzer = "standard"
searchAnalyzer = "standard"
}
}
keyword("genre")
objField("premiered") {
keyword("season")
number<Int>("year")
}
objField("aired") {
field("start", "date")
field("end", "date")
}
}
}
}
}
fun index() {
var i = 0;
entityStore {
val q = select(Anime::class)
indexRepo.bulk(50) {
for (item in q()) {
if (item.replacedWith != null) continue;
i++
if (i % 1_000 == 0) {
println(" => $i - ${item.id}")
}
index(
item.id.toString(),
AnimeDTO(
item.id,
item.titles.map {
AnimeTitleDTO(
it.name,
it.language,
it.type.toString(),
it.source
)
},
item.myAnimeListInfo?.let {
it.description?.let { desc ->
listOf(
AnimeDescriptionDTO(
desc,
InfoSource.MyAnimeList
)
)
}
} ?: listOf(),
item.genres.map { it.genre.name }.toSet().toList(),
item.myAnimeListInfo?.premiered,
item.myAnimeListInfo?.aired
),
false
)
}
}
}
println(" => Indexed $i entries.")
}
}

13
src/main/kotlin/moe/odango/index/es/dto/AnimeDTO.kt

@ -0,0 +1,13 @@
package moe.odango.index.es.dto
import moe.odango.index.scraper.mal.AnimePageScraper
import java.util.*
data class AnimeDTO(
val id: UUID,
val title: Collection<AnimeTitleDTO>,
val description: Collection<AnimeDescriptionDTO>,
val genre: Collection<String>,
val premiered: AnimePageScraper.Premiered?,
val aired: AnimePageScraper.Aired?
)

8
src/main/kotlin/moe/odango/index/es/dto/AnimeDescriptionDTO.kt

@ -0,0 +1,8 @@
package moe.odango.index.es.dto
import moe.odango.index.utils.InfoSource
data class AnimeDescriptionDTO(
val text: String,
val source: InfoSource
)

10
src/main/kotlin/moe/odango/index/es/dto/AnimeTitleDTO.kt

@ -0,0 +1,10 @@
package moe.odango.index.es.dto
import moe.odango.index.utils.InfoSource
data class AnimeTitleDTO(
val name: String,
val language: String?,
val type: String,
val source: InfoSource
)

41
src/main/kotlin/moe/odango/index/http/Server.kt

@ -0,0 +1,41 @@
package moe.odango.index.http
import com.expediagroup.graphql.SchemaGeneratorConfig
import com.expediagroup.graphql.TopLevelObject
import com.expediagroup.graphql.toSchema
import io.ktor.http.content.resource
import io.ktor.http.content.resources
import io.ktor.http.content.static
import io.ktor.routing.routing
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
import ktor.graphql.GraphQLRouteConfig
import ktor.graphql.graphQL
import moe.odango.index.http.graphql.AnimeService
class Server {
fun run() {
embeddedServer(Netty, port = 3336) {
routing {
graphQL("/graphql",toSchema(
SchemaGeneratorConfig(listOf("moe.odango.index")),
listOf(TopLevelObject(AnimeService()))
)) {
GraphQLRouteConfig(
graphiql = true
)
}
static {
resource("/", "web/index.html")
static("/") {
resources("web")
}
}
}
}
.start(true)
}
}

100
src/main/kotlin/moe/odango/index/http/graphql/AnimeService.kt

@ -0,0 +1,100 @@
package moe.odango.index.http.graphql
import io.inbot.eskotlinwrapper.IndexRepository
import io.inbot.eskotlinwrapper.dsl.*
import moe.odango.index.di
import moe.odango.index.entity.Title
import moe.odango.index.es.dto.AnimeDTO
import moe.odango.index.http.graphql.dto.AnimeItem
import moe.odango.index.http.graphql.dto.AnimeTitleItem
import moe.odango.index.http.graphql.dto.AutoCompleteItem
import moe.odango.index.utils.ScoreMode
import moe.odango.index.utils.nested
import org.elasticsearch.action.search.dsl
import org.elasticsearch.action.search.source
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.Fuzziness
import org.elasticsearch.search.suggest.SuggestBuilder
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder
import org.kodein.di.instance
class AnimeService {
private val es by di.instance<IndexRepository<AnimeDTO>>()
private val client by di.instance<RestHighLevelClient>()
fun autocomplete(title: String): List<AutoCompleteItem> {
return es.search {
source {
suggest(
SuggestBuilder().addSuggestion(
"title",
CompletionSuggestionBuilder("title.name").prefix(title, Fuzziness.AUTO)
)
)
}
}
.hits
.mapNotNull { (s, item) ->
AutoCompleteItem(
s.innerHits["title"]?.hits?.get(0)?.sourceAsMap?.get("name")?.toString() ?: return@mapNotNull null,
item?.id.toString() ?: return@mapNotNull null
)
}
.toList()
}
fun search(
title: String? = null,
description: String? = null,
genre: List<String>? = null
): List<AnimeItem> {
val items = mutableListOf<ESQuery>()
title?.let {
items.add(
nested("title") {
query = MatchQuery("title.name", it)
scoreMode = ScoreMode.max
}
)
}
genre?.let {
items.add(
bool {
must(*it.toSet().map { genre -> TermQuery("genre", genre) }.toTypedArray())
}
)
}
description?.let {
items.add(
nested("description") {
query = MatchQuery("description.name", it)
scoreMode = ScoreMode.max
}
)
}
return es.search {
dsl {
query = disMax {
queries(*items.toTypedArray())
}
}
}
.hits
.mapNotNull { (_, dto) ->
dto?.let {
AnimeItem(
it.id.toString(),
it.title.map { dto ->
AnimeTitleItem(dto.name, dto.language, dto.type.let(Title.TitleType::valueOf), dto.source)
},
it.genre.toSet().toList()
)
}
}
.toList()
}
}

7
src/main/kotlin/moe/odango/index/http/graphql/dto/AnimeItem.kt

@ -0,0 +1,7 @@
package moe.odango.index.http.graphql.dto
data class AnimeItem(
val id: String,
val titles: List<AnimeTitleItem>,
val genres: List<String>
)

11
src/main/kotlin/moe/odango/index/http/graphql/dto/AnimeTitleItem.kt

@ -0,0 +1,11 @@
package moe.odango.index.http.graphql.dto
import moe.odango.index.entity.Title
import moe.odango.index.utils.InfoSource
data class AnimeTitleItem(
val name: String,
val language: String?,
val type: Title.TitleType,
val source: InfoSource
)

6
src/main/kotlin/moe/odango/index/http/graphql/dto/AutoCompleteItem.kt

@ -0,0 +1,6 @@
package moe.odango.index.http.graphql.dto
data class AutoCompleteItem(
val title: String,
val id: String
)

30
src/main/kotlin/moe/odango/index/main.kt

@ -0,0 +1,30 @@
package moe.odango.index
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.subcommands
import com.github.ajalt.clikt.parameters.options.option
import moe.odango.index.cli.*
import org.kodein.di.instance
fun main(args: Array<String>) {
object : CliktCommand() {
val config: String? by option()
init {
subcommands(
AniDBSync(),
DatabaseMigration(),
MyAnimeListListingSync(),
MyAnimeListPageSync(),
HTTPServer(),
ElasticIndex()
)
}
override fun run() {
val configFile by di.instance<MutableList<String>>("config-file")
config?.let { configFile.add(0, it) }
}
}
.main(args)
}

33
src/main/kotlin/moe/odango/index/scraper/mal/AnimeListScraper.kt

@ -0,0 +1,33 @@
package moe.odango.index.scraper.mal
import moe.odango.index.entity.MyAnimeListInfo
import org.jsoup.Jsoup
import java.net.URI
class AnimeListScraper(body: String) {
private val dom = Jsoup.parse(body)
data class MyAnimeListListingItem(val myAnimeListId: Long, val title: String, val type: MyAnimeListInfo.ReleaseType, val episodes: Int?)
fun getItems(): List<MyAnimeListListingItem> {
val arr = dom
.select("[id=content] .list tbody tr")
.toList()
return arr.drop(1).mapNotNull {
val link = it.select("a[id^=sinfo]").first()
val id = link
.attr("href")
.split("/")
.let {
it[it.indexOf("anime") + 1].toLongOrNull()
}
val title = link.text()
val type = it.child(2).text()
val eps = it.child(3).text().toIntOrNull()
id?.let { MyAnimeListListingItem(id, title, MyAnimeListInfo.ReleaseType.valueOf(type), eps) }
}
}
}

246
src/main/kotlin/moe/odango/index/scraper/mal/AnimePageScraper.kt

@ -0,0 +1,246 @@
package moe.odango.index.scraper.mal
import moe.odango.index.entity.AnimeRelation
import moe.odango.index.entity.MyAnimeListInfo
import moe.odango.index.utils.IntDate
import moe.odango.index.utils.ProducerFunction
import moe.odango.index.utils.brText
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.time.Duration
import java.util.*
class AnimePageScraper(body: String) {
private val dom = Jsoup.parse(body)
private val items: MutableMap<String, Pair<String, Element>> = mutableMapOf()
data class Info(
val title: String,
val englishName: String?,
val japaneseName: String?,
val synonyms: List<String>,
val description: String,
val type: MyAnimeListInfo.ReleaseType,
val episodes: Int?,
val source: String?,
val image: String?,
val genres: List<Genre>,
val aired: Aired?,
val premiered: Premiered?,
val rating: MyAnimeListInfo.Rating?,
val duration: Duration?,
val related: List<Relation>,
val producers: List<ProducerRelation>
)
fun getInfo() = Info(
getTitle(),
getEnglishName(),
getJapaneseName(),
getSynonyms(),
getDescription(),
getReleaseType(),
getEpisodes(),
getSource(),
getImage(),
getGenres(),
getAired(),
getPremiered(),
getRating(),
getDuration(),
getRelated(),
getProducers()
)
fun getEpisodes(): Int? {
return items["episodes"]?.first?.toIntOrNull()
}
fun getReleaseType(): MyAnimeListInfo.ReleaseType {
return MyAnimeListInfo.ReleaseType.valueOf(items["type"]!!.first)
}
fun getTitle(): String {
return dom.select(".h1-title span[itemprop=name]").first().brText().split("\n").first()
}
fun getSynonyms(): List<String> {
// If the english name or title contains a comma we can't parse synonyms since they're splice by comma's
return if (getEnglishName()?.contains(",") == true || getTitle().contains(",")) listOf