You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
319 lines
11 KiB
Kotlin
319 lines
11 KiB
Kotlin
package moe.odango.index.sync
|
|
|
|
import io.requery.Persistable
|
|
import io.requery.kotlin.`in`
|
|
import io.requery.kotlin.eq
|
|
import io.requery.kotlin.invoke
|
|
import io.requery.sql.KotlinEntityDataStore
|
|
import moe.odango.index.di
|
|
import moe.odango.index.entity.*
|
|
import moe.odango.index.utils.InfoSource
|
|
import moe.odango.index.utils.MergeMap
|
|
import moe.odango.index.utils.brText
|
|
import org.jsoup.Jsoup
|
|
import org.jsoup.nodes.Element
|
|
import org.jsoup.parser.Parser
|
|
import org.kodein.di.instance
|
|
import java.io.File
|
|
import java.util.*
|
|
|
|
|
|
class AniDBXMLSync {
|
|
private val entityStore by di.instance<KotlinEntityDataStore<Persistable>>()
|
|
private val seriesToMerge = MergeMap<UUID>()
|
|
|
|
fun run(xmlDir: String) {
|
|
val files = File(xmlDir)
|
|
.listFiles()
|
|
.filter {
|
|
it.extension == "xml"
|
|
}
|
|
|
|
for (file in files) {
|
|
val xml = file.readText(Charsets.UTF_8)
|
|
indexAniDBEntry(xml)
|
|
}
|
|
}
|
|
|
|
private fun indexAniDBEntry(xml: String) {
|
|
val doc = Jsoup.parse(xml, "", Parser.xmlParser())
|
|
val anime = doc.selectFirst("anime")
|
|
val aid = anime.attr("id").toLongOrNull() ?: return
|
|
|
|
val animeEnt: Anime? = entityStore {
|
|
val q = select(Anime::class) where (Anime::aniDbId eq aid)
|
|
q().firstOrNull()
|
|
}
|
|
|
|
val myAnimeListId = anime
|
|
.selectFirst("resource[type=2] externalentity identifier")
|
|
?.text()
|
|
?.toLongOrNull()
|
|
|
|
val malAnimeEnt = myAnimeListId?.let {
|
|
entityStore {
|
|
val q = select(Anime::class) where (Anime::myAnimeListId eq myAnimeListId)
|
|
|
|
q().firstOrNull()
|
|
}
|
|
}
|
|
|
|
if (malAnimeEnt != null && animeEnt != null) {
|
|
// WE GOT ISSUES OH NO
|
|
TODO()
|
|
}
|
|
|
|
val entity = malAnimeEnt ?: animeEnt ?: Anime {
|
|
this.myAnimeListId = myAnimeListId
|
|
this.aniDbId = aid
|
|
}.let { entityStore.insert(it) }
|
|
|
|
val aniDbInfo = entity.aniDbInfo ?: AniDBInfo {
|
|
setAnime(entity)
|
|
}.let { entityStore.insert(it) }
|
|
|
|
aniDbInfo.restricted = anime.attr("restricted") == "true"
|
|
aniDbInfo.description = doc.selectFirst("anime > description")?.brText() ?: ""
|
|
aniDbInfo.episodes = doc.selectFirst("anime > episodecount")?.text()?.trim()?.toIntOrNull()
|
|
aniDbInfo.type =
|
|
doc.selectFirst("anime > type")?.text()?.trim()?.let { AniDBInfo.ReleaseType.fromAniDBString(it) }
|
|
?: AniDBInfo.ReleaseType.Unknown
|
|
aniDbInfo.image = doc.selectFirst("anime > picture")?.text()?.trim()
|
|
val startDate = doc.selectFirst("anime > startdate")?.text()?.trim()?.let {
|
|
val items = it.split("-")
|
|
if (items.size > 3) {
|
|
null
|
|
} else {
|
|
items.map { item -> item.toIntOrNull() }
|
|
}
|
|
} ?: emptyList()
|
|
|
|
aniDbInfo.startYear = startDate[0]
|
|
aniDbInfo.startMonth = startDate[1]
|
|
aniDbInfo.startDay = startDate[2]
|
|
|
|
val endDate = doc.selectFirst("anime > enddate")?.text()?.trim()?.let {
|
|
val items = it.split("-")
|
|
if (items.size > 3) {
|
|
null
|
|
} else {
|
|
items.map { item -> item.toIntOrNull() }
|
|
}
|
|
} ?: emptyList()
|
|
|
|
aniDbInfo.endYear = endDate[0]
|
|
aniDbInfo.endMonth = endDate[1]
|
|
aniDbInfo.endDay = endDate[2]
|
|
|
|
entityStore.update(aniDbInfo)
|
|
|
|
val aniDbTitles = entityStore {
|
|
val q = select(Title::class) where (Title::anime eq entity) and (Title::source eq InfoSource.AniDB)
|
|
q().toList()
|
|
}
|
|
|
|
.associateBy { "${it.type}/${it.language}/${it.name}" }
|
|
.toMutableMap()
|
|
|
|
for (title in doc.select("anime > titles > title")) {
|
|
val name = title.text() ?: continue
|
|
val lang = title.attr("lang") ?: null
|
|
val type = title.attr("type")?.let { Title.TitleType.fromAniDBString(it) } ?: Title.TitleType.Synonym
|
|
val handle = "$type/$lang/$name"
|
|
|
|
if (aniDbTitles.remove(handle) == null) {
|
|
entityStore.insert(Title {
|
|
setAnime(entity)
|
|
setSource(InfoSource.AniDB)
|
|
setLanguage(lang)
|
|
setType(type)
|
|
|
|
this.name = name
|
|
})
|
|
}
|
|
}
|
|
|
|
for ((_, title) in aniDbTitles) {
|
|
entityStore.delete(title)
|
|
}
|
|
|
|
val relatedAnimeFrom = entityStore {
|
|
val q =
|
|
select(AnimeRelation::class) where (AnimeRelation::from eq entity) and (AnimeRelation::source eq InfoSource.AniDB)
|
|
q().toList()
|
|
}
|
|
.associateBy { it.to.aniDbId!! }
|
|
.toMutableMap()
|
|
|
|
val relatedAnimeTo = entityStore {
|
|
val q =
|
|
select(AnimeRelation::class) where (AnimeRelation::to eq entity) and (AnimeRelation::source eq InfoSource.AniDB)
|
|
q().toList()
|
|
}
|
|
.associateBy { it.from.aniDbId!! }
|
|
.toMutableMap()
|
|
|
|
for (related in doc.select("anime > relatedanime > anime")) {
|
|
val id = related.attr("id").toLongOrNull() ?: continue
|
|
val relatedAnime = entityStore {
|
|
val q = select(Anime::class) where (Anime::aniDbId eq id)
|
|
q().firstOrNull()
|
|
} ?: Anime { aniDbId = id }.let { entityStore.insert(it) }
|
|
|
|
val type = related.attr("type")?.let { AnimeRelation.RelationType.fromString(it) }
|
|
?: AnimeRelation.RelationType.Other
|
|
|
|
val currentFromRelation = relatedAnimeFrom.remove(id)
|
|
if (currentFromRelation == null) {
|
|
entityStore.insert(AnimeRelation {
|
|
setFrom(entity)
|
|
setTo(relatedAnime)
|
|
setSource(InfoSource.AniDB)
|
|
relation = type
|
|
})
|
|
} else if (currentFromRelation.relation != type) {
|
|
currentFromRelation.relation = type
|
|
entityStore.update(currentFromRelation)
|
|
}
|
|
|
|
val currentToRelation = relatedAnimeTo.remove(id)
|
|
if (currentToRelation == null) {
|
|
entityStore.insert(AnimeRelation {
|
|
setFrom(relatedAnime)
|
|
setTo(entity)
|
|
setSource(InfoSource.AniDB)
|
|
relation = type.inverse
|
|
})
|
|
}
|
|
|
|
// Character only have characters of that anime in the other anime
|
|
// So are not part of The Series, see e.g. Isekai Quartet
|
|
if (type != AnimeRelation.RelationType.Character) {
|
|
if (relatedAnime.series != null && entity.series == null) {
|
|
entity.series = relatedAnime.series
|
|
entityStore.update(entity)
|
|
} else if (relatedAnime.series == null && entity.series != null) {
|
|
relatedAnime.series = entity.series
|
|
entityStore.update(relatedAnime)
|
|
} else if (relatedAnime.series == null && entity.series == null) {
|
|
val newSeries = AnimeSeries {}
|
|
entityStore.insert(newSeries)
|
|
entity.series = newSeries
|
|
entityStore.update(entity)
|
|
relatedAnime.series = newSeries
|
|
entityStore.update(relatedAnime)
|
|
} else if (relatedAnime.series != null && entity.series != null) {
|
|
seriesToMerge.add(relatedAnime.series!!.id, entity.series!!.id)
|
|
}
|
|
}
|
|
}
|
|
|
|
val tags = entityStore {
|
|
val q =
|
|
select(AnimeTag::class) join Tag::class on (AnimeTag::tag eq Tag::class) where (AnimeTag::anime eq entity) and (AnimeTag::source eq InfoSource.AniDB)
|
|
q().toList()
|
|
}
|
|
.associateBy { it.tag.aniDbId!! }
|
|
.toMutableMap()
|
|
|
|
val tagIds = doc.select("anime > tags > tag").flatMap {
|
|
val ids = mutableSetOf<Long>()
|
|
it.attr("id")?.toLongOrNull()?.let(ids::add)
|
|
it.attr("parentid")?.toLongOrNull()?.let(ids::add)
|
|
|
|
ids
|
|
}
|
|
.toSet()
|
|
|
|
val tagsById = entityStore {
|
|
val q = select(Tag::class) where (Tag::aniDbId `in` tagIds)
|
|
q().toList()
|
|
}
|
|
.associateBy { it.aniDbId!! }
|
|
.toMutableMap()
|
|
|
|
val after = mutableListOf<Element>()
|
|
|
|
fun getTag(tag: Element): Tag {
|
|
val tagId = tag.attr("id")?.toLongOrNull()!!
|
|
val name = tag.selectFirst("name")?.text()?.trim()!!
|
|
val description = tag.selectFirst("description")?.brText() ?: ""
|
|
val parentId = tag.attr("parentid")?.toLongOrNull()
|
|
|
|
return tagsById.getOrPut(tagId) {
|
|
val newTag = Tag {
|
|
setAniDbId(tagId)
|
|
setName(name)
|
|
setDescription(description)
|
|
setSpoiler(tag.attr("globalspoiler") == "true")
|
|
parentId?.let {
|
|
setParentId(tagsById[it]!!.id)
|
|
}
|
|
}
|
|
|
|
entityStore.insert(newTag)
|
|
newTag
|
|
}
|
|
}
|
|
|
|
fun ensureTag(tag: Element, tagId: Long) {
|
|
if (tags.remove(tagId) == null) {
|
|
val tagEntity = getTag(tag)
|
|
val animeTag = AnimeTag {
|
|
setTag(tagEntity)
|
|
setAnime(entity)
|
|
setSpoiler(tag.attr("localspoiler") == "true")
|
|
setSource(InfoSource.AniDB)
|
|
}
|
|
|
|
entityStore.insert(animeTag)
|
|
}
|
|
}
|
|
|
|
for (tag in doc.select("anime > tags > tag")) {
|
|
val tagId = tag.attr("id")?.toLongOrNull() ?: continue
|
|
val name = tag.selectFirst("name")?.text()?.trim() ?: continue
|
|
val parentId = tag.attr("parentid")?.toLongOrNull()
|
|
|
|
if (parentId != null && !tagsById.contains(parentId)) {
|
|
after.add(tag)
|
|
continue
|
|
}
|
|
|
|
ensureTag(tag, tagId)
|
|
}
|
|
|
|
for (tag in after) {
|
|
val tagId = tag.attr("id")?.toLongOrNull() ?: continue
|
|
ensureTag(tag, tagId)
|
|
}
|
|
|
|
for ((_, tag) in tags) {
|
|
entityStore.delete(tag)
|
|
}
|
|
}
|
|
|
|
fun mergeSeries() {
|
|
for (coll in seriesToMerge) {
|
|
val items = coll.toMutableSet()
|
|
val first = coll.first()
|
|
items.remove(first)
|
|
|
|
entityStore {
|
|
val u = update(AnimeSeries::class)
|
|
.set(AnimeSeriesEntity.REPLACED_WITH, first)
|
|
.where((AnimeSeries::id `in` items) or (AnimeSeries::replacedWith `in` items))
|
|
|
|
u()
|
|
|
|
val u2 = update(Anime::class)
|
|
.set(AnimeEntity.SERIES_ID, first)
|
|
.where(Anime::series `in` items)
|
|
|
|
u2()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|