package moe.odango.index.sync import io.requery.Persistable import io.requery.kotlin.`in` import io.requery.kotlin.eq import io.requery.kotlin.invoke import io.requery.sql.KotlinEntityDataStore import moe.odango.index.di import moe.odango.index.entity.* import moe.odango.index.utils.InfoSource import moe.odango.index.utils.MergeMap import moe.odango.index.utils.brText import org.jsoup.Jsoup import org.jsoup.nodes.Element import org.jsoup.parser.Parser import org.kodein.di.instance import java.io.File import java.util.* class AniDBXMLSync { private val entityStore by di.instance>() private val seriesToMerge = MergeMap() fun run(xmlDir: String) { val files = File(xmlDir) .listFiles() .filter { it.extension == "xml" } for (file in files) { val xml = file.readText(Charsets.UTF_8) indexAniDBEntry(xml) } } private fun indexAniDBEntry(xml: String) { val doc = Jsoup.parse(xml, "", Parser.xmlParser()) val anime = doc.selectFirst("anime") val aid = anime.attr("id").toLongOrNull() ?: return val animeEnt: Anime? = entityStore { val q = select(Anime::class) where (Anime::aniDbId eq aid) q().firstOrNull() } val myAnimeListId = anime .selectFirst("resource[type=2] externalentity identifier") ?.text() ?.toLongOrNull() val malAnimeEnt = myAnimeListId?.let { entityStore { val q = select(Anime::class) where (Anime::myAnimeListId eq myAnimeListId) q().firstOrNull() } } if (malAnimeEnt != null && animeEnt != null) { // WE GOT ISSUES OH NO TODO() } val entity = malAnimeEnt ?: animeEnt ?: Anime { this.myAnimeListId = myAnimeListId this.aniDbId = aid }.let { entityStore.insert(it) } val aniDbInfo = entity.aniDbInfo ?: AniDBInfo { setAnime(entity) }.let { entityStore.insert(it) } aniDbInfo.restricted = anime.attr("restricted") == "true" aniDbInfo.description = doc.selectFirst("anime > description")?.brText() ?: "" aniDbInfo.episodes = doc.selectFirst("anime > episodecount")?.text()?.trim()?.toIntOrNull() aniDbInfo.type = doc.selectFirst("anime > type")?.text()?.trim()?.let { AniDBInfo.ReleaseType.fromAniDBString(it) } ?: AniDBInfo.ReleaseType.Unknown aniDbInfo.image = doc.selectFirst("anime > picture")?.text()?.trim() val startDate = doc.selectFirst("anime > startdate")?.text()?.trim()?.let { val items = it.split("-") if (items.size > 3) { null } else { items.map { item -> item.toIntOrNull() } } } ?: emptyList() aniDbInfo.startYear = startDate[0] aniDbInfo.startMonth = startDate[1] aniDbInfo.startDay = startDate[2] val endDate = doc.selectFirst("anime > enddate")?.text()?.trim()?.let { val items = it.split("-") if (items.size > 3) { null } else { items.map { item -> item.toIntOrNull() } } } ?: emptyList() aniDbInfo.endYear = endDate[0] aniDbInfo.endMonth = endDate[1] aniDbInfo.endDay = endDate[2] entityStore.update(aniDbInfo) val aniDbTitles = entityStore { val q = select(Title::class) where (Title::anime eq entity) and (Title::source eq InfoSource.AniDB) q().toList() } .associateBy { "${it.type}/${it.language}/${it.name}" } .toMutableMap() for (title in doc.select("anime > titles > title")) { val name = title.text() ?: continue val lang = title.attr("lang") ?: null val type = title.attr("type")?.let { Title.TitleType.fromAniDBString(it) } ?: Title.TitleType.Synonym val handle = "$type/$lang/$name" if (aniDbTitles.remove(handle) == null) { entityStore.insert(Title { setAnime(entity) setSource(InfoSource.AniDB) setLanguage(lang) setType(type) this.name = name }) } } for ((_, title) in aniDbTitles) { entityStore.delete(title) } val relatedAnimeFrom = entityStore { val q = select(AnimeRelation::class) where (AnimeRelation::from eq entity) and (AnimeRelation::source eq InfoSource.AniDB) q().toList() } .associateBy { it.to.aniDbId!! } .toMutableMap() val relatedAnimeTo = entityStore { val q = select(AnimeRelation::class) where (AnimeRelation::to eq entity) and (AnimeRelation::source eq InfoSource.AniDB) q().toList() } .associateBy { it.from.aniDbId!! } .toMutableMap() for (related in doc.select("anime > relatedanime > anime")) { val id = related.attr("id").toLongOrNull() ?: continue val relatedAnime = entityStore { val q = select(Anime::class) where (Anime::aniDbId eq id) q().firstOrNull() } ?: Anime { aniDbId = id }.let { entityStore.insert(it) } val type = related.attr("type")?.let { AnimeRelation.RelationType.fromString(it) } ?: AnimeRelation.RelationType.Other val currentFromRelation = relatedAnimeFrom.remove(id) if (currentFromRelation == null) { entityStore.insert(AnimeRelation { setFrom(entity) setTo(relatedAnime) setSource(InfoSource.AniDB) relation = type }) } else if (currentFromRelation.relation != type) { currentFromRelation.relation = type entityStore.update(currentFromRelation) } val currentToRelation = relatedAnimeTo.remove(id) if (currentToRelation == null) { entityStore.insert(AnimeRelation { setFrom(relatedAnime) setTo(entity) setSource(InfoSource.AniDB) relation = type.inverse }) } // Character only have characters of that anime in the other anime // So are not part of The Series, see e.g. Isekai Quartet if (type != AnimeRelation.RelationType.Character) { if (relatedAnime.series != null && entity.series == null) { entity.series = relatedAnime.series entityStore.update(entity) } else if (relatedAnime.series == null && entity.series != null) { relatedAnime.series = entity.series entityStore.update(relatedAnime) } else if (relatedAnime.series == null && entity.series == null) { val newSeries = AnimeSeries {} entityStore.insert(newSeries) entity.series = newSeries entityStore.update(entity) relatedAnime.series = newSeries entityStore.update(relatedAnime) } else if (relatedAnime.series != null && entity.series != null) { seriesToMerge.add(relatedAnime.series!!.id, entity.series!!.id) } } } val tags = entityStore { val q = select(AnimeTag::class) join Tag::class on (AnimeTag::tag eq Tag::class) where (AnimeTag::anime eq entity) and (AnimeTag::source eq InfoSource.AniDB) q().toList() } .associateBy { it.tag.aniDbId!! } .toMutableMap() val tagIds = doc.select("anime > tags > tag").flatMap { val ids = mutableSetOf() it.attr("id")?.toLongOrNull()?.let(ids::add) it.attr("parentid")?.toLongOrNull()?.let(ids::add) ids } .toSet() val tagsById = entityStore { val q = select(Tag::class) where (Tag::aniDbId `in` tagIds) q().toList() } .associateBy { it.aniDbId!! } .toMutableMap() val after = mutableListOf() fun getTag(tag: Element): Tag { val tagId = tag.attr("id")?.toLongOrNull()!! val name = tag.selectFirst("name")?.text()?.trim()!! val description = tag.selectFirst("description")?.brText() ?: "" val parentId = tag.attr("parentid")?.toLongOrNull() return tagsById.getOrPut(tagId) { val newTag = Tag { setAniDbId(tagId) setName(name) setDescription(description) setSpoiler(tag.attr("globalspoiler") == "true") parentId?.let { setParentId(tagsById[it]!!.id) } } entityStore.insert(newTag) newTag } } fun ensureTag(tag: Element, tagId: Long) { if (tags.remove(tagId) == null) { val tagEntity = getTag(tag) val animeTag = AnimeTag { setTag(tagEntity) setAnime(entity) setSpoiler(tag.attr("localspoiler") == "true") setSource(InfoSource.AniDB) } entityStore.insert(animeTag) } } for (tag in doc.select("anime > tags > tag")) { val tagId = tag.attr("id")?.toLongOrNull() ?: continue val name = tag.selectFirst("name")?.text()?.trim() ?: continue val parentId = tag.attr("parentid")?.toLongOrNull() if (parentId != null && !tagsById.contains(parentId)) { after.add(tag) continue } ensureTag(tag, tagId) } for (tag in after) { val tagId = tag.attr("id")?.toLongOrNull() ?: continue ensureTag(tag, tagId) } for ((_, tag) in tags) { entityStore.delete(tag) } } fun mergeSeries() { for (coll in seriesToMerge) { val items = coll.toMutableSet() val first = coll.first() items.remove(first) entityStore { val u = update(AnimeSeries::class) .set(AnimeSeriesEntity.REPLACED_WITH, first) .where((AnimeSeries::id `in` items) or (AnimeSeries::replacedWith `in` items)) u() val u2 = update(Anime::class) .set(AnimeEntity.SERIES_ID, first) .where(Anime::series `in` items) u2() } } } }