package moe.odango.index.sync import com.github.kittinunf.fuel.coroutines.awaitStringResult import com.github.kittinunf.fuel.httpGet import io.requery.Persistable import io.requery.kotlin.`in` import io.requery.kotlin.eq import io.requery.kotlin.invoke import io.requery.sql.KotlinEntityDataStore import moe.odango.index.di import moe.odango.index.entity.Anime import moe.odango.index.entity.MyAnimeListInfo import moe.odango.index.entity.Title import moe.odango.index.scraper.mal.AnimeListScraper import moe.odango.index.utils.InfoSource import org.kodein.di.instance import java.util.concurrent.TimeUnit class MyAnimeListListingSync : ScheduledSync(2, TimeUnit.DAYS) { val entityStore: KotlinEntityDataStore by di.instance() override suspend fun run() { var offset = 0 do { println(" => MAL Offset $offset") val items = getListing(offset) val animes = entityStore { val q = select(Anime::class) where (Anime::myAnimeListId `in` items.map { it.myAnimeListId }) q().toList() }.associateBy { it.myAnimeListId!! } val infos = entityStore { val q = select(MyAnimeListInfo::class) join Anime::class on (MyAnimeListInfo::anime `in` animes.values) q().toList() }.associateBy { it.anime.myAnimeListId!! } val titles = entityStore { val q = select(Title::class) where (Title::source eq InfoSource.MyAnimeList) and (Title::language eq "en") and (Title::anime `in` animes.values) q().toList() }.associateBy { it.anime.myAnimeListId!! } entityStore.withTransaction { for (item in items) { println("Syncing MAL#${item.myAnimeListId} ${item.title}") val anime = animes[item.myAnimeListId] ?: run { val newAnime = Anime { myAnimeListId = item.myAnimeListId } insert(newAnime) newAnime } val title = titles[item.myAnimeListId] ?: run { val newTitle = Title { setAnime(anime) setLanguage("en") setType(Title.TitleType.Official) setSource(InfoSource.MyAnimeList) name = item.title } insert(newTitle) newTitle } if (title.name != item.title) { title.name = item.title update(title) } val info = infos[item.myAnimeListId] ?: run { val newInfo = MyAnimeListInfo { setAnime(anime) episodes = item.episodes releaseType = item.type } insert(newInfo) newInfo } if (info.episodes != item.episodes || info.releaseType != item.type) { info.episodes = item.episodes info.releaseType = item.type update(info) } } } offset += items.size } while (items.size == 50) } suspend fun getListing(offset: Int): List { val url = "https://myanimelist.net/anime.php?o=9&c[0]=a&c[1]=b&cv=2&w=1&show=$offset" val body = url .httpGet() .awaitStringResult() .get() return AnimeListScraper(body).getItems() } }