package moe.odango.index.sync import com.github.kittinunf.fuel.coroutines.awaitByteArrayResponseResult import com.github.kittinunf.fuel.httpDownload import io.requery.Persistable import io.requery.kotlin.`in` import io.requery.kotlin.eq import io.requery.kotlin.invoke import io.requery.sql.KotlinEntityDataStore import moe.odango.index.di import moe.odango.index.entity.Anime import moe.odango.index.entity.Title import moe.odango.index.utils.InfoSource import moe.odango.index.utils.XMLOutputStreamReader import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.kodein.di.instance import java.io.File import java.io.InputStream import java.util.concurrent.TimeUnit import java.util.zip.InflaterOutputStream class AniDBTitleSync : ScheduledSync(1, TimeUnit.DAYS) { private val entityStore by di.instance>() data class AniDBTitle(val aid: Long, val name: String, val type: String, val language: String) override suspend fun run() { val (feeder, cacheMap) = getFeeder() "https://anidb.net/api/anime-titles.xml.gz" .httpDownload() .streamDestination { _, _ -> InflaterOutputStream(feeder) to { InputStream.nullInputStream() } } .awaitByteArrayResponseResult() syncTitles(cacheMap) } private fun getFeeder(): Pair>> { var animeId: Long = 0; var language = "" var type = "" var title = "" val cacheMap = mutableMapOf>() return XMLOutputStreamReader { if (isStartElement) { if (name.localPart == "anime") { animeId = getAttributeValue("", "aid").toLong() } if (name.localPart == "title") { title = "" type = getAttributeValue("", "type") language = getAttributeValue("http://www.w3.org/XML/1998/namespace", "lang") } } if (isCharacters) { title += text } if (isEndElement) { if (name.localPart == "title") { cacheMap .getOrPut(animeId, ::mutableListOf) .add(AniDBTitle(animeId, title, type, language)) println("$animeId => [$type/$language] $title") title = "" } if (name.localPart == "anime") { if (cacheMap.size >= 100) { syncTitles(cacheMap) cacheMap.clear() } } } } to cacheMap } fun syncWithFile(file: File) { val (feeder, cacheMap) = getFeeder() file .inputStream() .let { if (file.name.endsWith(".gz")) { GzipCompressorInputStream(it) } else { it } } .transferTo(feeder) syncTitles(cacheMap) } private fun syncTitles(cacheMap: Map>) { val ids = cacheMap.keys val (animes, titles) = entityStore { val animes = (select(Anime::class) where (Anime::aniDbId `in` ids)).get().toList() val query = select(Title::class) where (Title::anime `in` animes) and (Title::source eq InfoSource.AniDB) animes to query().toList() } val animeById = animes.associateBy { it.aniDbId!! } val titlesByAniDBId = titles.groupBy { it.anime.aniDbId!! } val newAnimes = mutableListOf>>() entityStore.withTransaction { for ((aniDb, newTitles) in cacheMap) { val newAniDbTitlesSet = newTitles.map { Triple(it.language, it.type, it.name) }.toMutableSet() val anime = animeById[aniDb] if (anime == null) { newAnimes.add(aniDb to newTitles) continue } val dbTitles = titlesByAniDBId[aniDb] ?: listOf() for (dbTitle in dbTitles) { if (!newAniDbTitlesSet.remove( Triple( dbTitle.language, dbTitle.type.toAniDBString(), dbTitle.name ) ) ) { // If it wasn't in the set, remove it from the DB delete(dbTitle) } } for ((language, type, name) in newAniDbTitlesSet) { val title = Title { setAnime(anime) setLanguage(language) setType(Title.TitleType.fromAniDBString(type)) setSource(InfoSource.AniDB) this.name = name } insert(title) } } for ((aniDb, aniDbTitles) in newAnimes) { val anime = Anime { aniDbId = aniDb } insert(anime) for (aniDbTitle in aniDbTitles) { insert(Title { setAnime(anime) setLanguage(aniDbTitle.language) setType(Title.TitleType.fromAniDBString(aniDbTitle.type)) setSource(InfoSource.AniDB) name = aniDbTitle.name }) } } } } }