Initial commit (0dd4d6f1) · Commits · Kurikku / backend / ripple-cron-go

.gitignore

0 → 100644

+2 −0

Changes for .gitignore: 2 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		ripple-cron-go
		cron.conf
		No newline at end of file

cache_data.go

0 → 100644

+178 −0

Changes for cache_data.go: 178 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"fmt"
		"math"
		"strconv"
		)

		type s struct {
		rankedScore int64
		totalHits int64
		level int
		}

		func opCacheData() {
		// get data
		const fetchQuery = `SELECT users.id as user_id, users.username, scores.play_mode, scores.score, scores.completed, scores.300_count, scores.100_count, scores.50_count FROM scores LEFT JOIN users ON users.username=scores.username WHERE users.allowed = '1'`
		rows, err := db.Query(fetchQuery)
		if err != nil {
		queryError(err, fetchQuery)
		return
		}

		// set up end map where all the data is
		data := make(map[int][4]s)

		count := 0

		// analyse every result row of fetchQuery
		for rows.Next() {
		if count%1000 == 0 {
		fmt.Println("> CacheData:", count)
		}
		var (
		uid int
		username string
		playMode int
		score int64
		completed int
		count300 int
		count100 int
		count50 int
		)
		err := rows.Scan(&uid, &username, &playMode, &score, &completed, &count300, &count100, &count50)
		if err != nil {
		queryError(err, fetchQuery)
		continue
		}
		// silently ignore invalid modes
		if playMode > 3 \|\| playMode < 0 {
		continue
		}
		// create key in map if not already existing
		if _, ex := data[uid]; !ex {
		data[uid] = &[4]*s{}
		for i := 0; i < 4; i++ {
		data[uid][i] = &s{}
		}
		}
		// if the score counts as completed and top score, add it to the ranked score sum
		if c.CacheRankedScore && completed == 3 {
		data[uid][playMode].rankedScore += score
		}
		// add to the number of totalhits count of {300,100,50} hits
		if c.CacheTotalHits {
		data[uid][playMode].totalHits += int64(count300) + int64(count100) + int64(count50)
		}
		count++
		}
		rows.Close()

		if c.CacheLevel {
		const totalScoreQuery = "SELECT id, total_score_std, total_score_taiko, total_score_ctb, total_score_mania FROM users_stats"
		rows, err := db.Query(totalScoreQuery)
		if err != nil {
		queryError(err, totalScoreQuery)
		return
		}
		count = 0
		for rows.Next() {
		if count%100 == 0 {
		fmt.Println("> CacheLevel:", count)
		}
		var (
		id int
		std int64
		taiko int64
		ctb int64
		mania int64
		)
		err := rows.Scan(&id, &std, &taiko, &ctb, &mania)
		if err != nil {
		queryError(err, totalScoreQuery)
		continue
		}
		if _, ex := data[id]; !ex {
		data[id] = &[4]*s{}
		for i := 0; i < 4; i++ {
		data[id][i] = &s{}
		}
		}
		data[id][0].level = getLevel(std)
		data[id][1].level = getLevel(taiko)
		data[id][2].level = getLevel(ctb)
		data[id][3].level = getLevel(mania)
		count++
		}
		rows.Close()
		}
		for k, v := range data {
		if v == nil {
		continue
		}
		for modeInt, modeData := range v {
		if modeData == nil {
		continue
		}
		var setQ string
		var params []interface{}
		if c.CacheRankedScore {
		setQ += "ranked_score_" + modeToString(modeInt) + " = ?"
		params = append(params, (*modeData).rankedScore)
		}
		if c.CacheTotalHits {
		if setQ != "" {
		setQ += ", "
		}
		setQ += "total_hits_" + modeToString(modeInt) + " = ?"
		params = append(params, (*modeData).totalHits)
		}
		if c.CacheLevel {
		if setQ != "" {
		setQ += ", "
		}
		setQ += "level_" + modeToString(modeInt) + " = ?"
		params = append(params, (*modeData).level)
		}
		if setQ != "" {
		params = append(params, k)
		op("UPDATE users_stats SET "+setQ+" WHERE id = ?", params...)
		}
		}
		}
		wg.Done()
		}

		func getLevel(rankedScore int64) int {
		for i := 1; i < 8000; i++ {
		lScore := getRequiredScoreForLevel(i)
		if rankedScore < lScore {
		return i
		}
		}
		return 8000
		}
		func getRequiredScoreForLevel(level int) int64 {
		if level <= 100 {
		if level > 1 {
		return int64(math.Floor(float64(500)/3(4math.Pow(float64(level), 3)-3math.Pow(float64(level), 2)-float64(level)) + math.Floor(1.25math.Pow(1.8, float64(level)-60))))
		}
		return 1
		}
		return 26931190829 + 100000000000*int64(level-100)
		}

		var modes = [...]string{
		"std",
		"taiko",
		"ctb",
		"mania",
		}

		func modeToString(modeID int) string {
		if modeID < len(modes) {
		return modes[modeID]
		}
		return strconv.Itoa(modeID)
		}

calculate_accuracy.go

0 → 100644

+73 −0

Changes for calculate_accuracy.go: 73 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"fmt"
		"math"
		)

		func opCalculateAccuracy() {
		const initQuery = "SELECT id, 300_count, 100_count, 50_count, gekis_count, katus_count, misses_count, play_mode, accuracy FROM scores"
		rows, err := db.Query(initQuery)
		if err != nil {
		queryError(err, initQuery)
		}
		count := 0
		for rows.Next() {
		if count%1000 == 0 {
		fmt.Println("> CalculateAccuracy:", count)
		}
		var (
		id int
		count300 int
		count100 int
		count50 int
		countgeki int
		countkatu int
		countmiss int
		playMode int
		accuracy *float64
		)
		err := rows.Scan(&id, &count300, &count100, &count50, &countgeki, &countkatu, &countmiss, &playMode, &accuracy)
		if err != nil {
		queryError(err, initQuery)
		continue
		}
		if accuracy == nil {
		var a float64
		accuracy = &a
		}
		newAcc := calculateAccuracy(count300, count100, count50, countgeki, countkatu, countmiss, playMode)
		// if accuracies are not accurate to the .001
		if !math.IsNaN(newAcc) && math.Floor(newAcc1000) != math.Floor((accuracy)*1000) {
		op("UPDATE scores SET accuracy = ? WHERE id = ?", newAcc, id)
		}
		count++
		}
		rows.Close()
		wg.Done()
		}

		func calculateAccuracy(count300, count100, count50, countgeki, countkatu, countmiss, playMode int) float64 {
		var accuracy float64
		switch playMode {
		case 1:
		// Please note this is not what is written on the wiki.
		// However, what was written on the wiki didn't make any sense at all.
		totalPoints := (count10050 + count300100)
		maxHits := (countmiss + count100 + count300)
		accuracy = float64(totalPoints) / float64(maxHits*100)
		case 2:
		fruits := count300 + count100 + count50
		totalFruits := fruits + countmiss + countkatu
		accuracy = float64(fruits) / float64(totalFruits)
		case 3:
		totalPoints := (count5050 + count100100 + countkatu200 + count300300 + countgeki*300)
		maxHits := (countmiss + count50 + count100 + count300 + countgeki + countkatu)
		accuracy = float64(totalPoints) / float64(maxHits*300)
		default:
		totalPoints := (count5050 + count100100 + count300*300)
		maxHits := (countmiss + count50 + count100 + count300)
		accuracy = float64(totalPoints) / float64(maxHits*300)
		}
		return accuracy * 100
		}

cron.go

0 → 100644

+156 −0

Changes for cron.go: 156 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"database/sql"
		"fmt"
		"sync"

		"github.com/fatih/color"
		_ "github.com/go-sql-driver/mysql"
		"github.com/thehowl/conf"
		)

		type config struct {
		DSN string
		RippleDir string `description:"The ripple folder (e.g. /var/www/ripple, NOT /var/www/ripple/osu.ppy.sh). Write the directory relatively to where the ripple-cron-go executable is placed."`

		CalculateAccuracy bool
		CacheRankedScore bool
		CacheTotalHits bool
		CacheLevel bool

		DeleteOldPasswordResets bool
		CleanReplays bool
		DeleteReplayCache bool
		BuildLeaderboards bool

		LogQueries bool `description:"You don't wanna do this in prod."`
		}

		var db *sql.DB
		var c config
		var wg sync.WaitGroup
		var chanWg sync.WaitGroup

		func main() {
		// Set up the configuration.
		err := conf.Load(&c, "cron.conf")
		switch {
		case err == conf.ErrNoFile:
		color.Yellow("No cron.conf was found. Creating it...")
		err := conf.Export(&c, "cron.conf")
		if err != nil {
		color.Red("Couldn't create cron.conf: %v.", err)
		} else {
		color.Green("cron.conf has been created!")
		}
		return
		case err != nil:
		color.Red("cron.conf couldn't be loaded: %v.", err)
		return
		}

		fmt.Println(`
		___ _ __ ___ _ __
		/ __\| '__/ _ \\| '_ \
		\| (__\| \| \| (_) \| \| \| \|
		\___\|_\| \___/\|_\| \|_\|
		`)
		color.Green(" (not so) proudly brought to you by")
		color.Green(" The Ripple Team™")
		fmt.Println()

		fmt.Print("Starting MySQL connection...")
		// start database connection
		db, err = sql.Open("mysql", c.DSN)
		if err != nil {
		color.Red(" couldn't start MySQL connection: %v", err)
		return
		}
		color.Green(" ok!")
		defer db.Close()

		// spawn 8 workers
		fmt.Print("Spawning necessary workers...")
		for i := 0; i < 8; i++ {
		chanWg.Add(1)
		go worker()
		}
		color.Green(" ok!")

		if c.CalculateAccuracy {
		fmt.Print("Starting accuracy calculator worker...")
		wg.Add(1)
		go opCalculateAccuracy()
		color.Green(" ok!")
		}
		if c.DeleteOldPasswordResets {
		fmt.Print("Starting deleting old password resets...")
		go op("DELETE FROM password_recovery WHERE t < (NOW() - INTERVAL 10 DAY);")
		color.Green(" ok!")
		}
		if c.CacheLevel \|\| c.CacheTotalHits \|\| c.CacheRankedScore {
		fmt.Print("Starting caching of various user stats...")
		wg.Add(1)
		go opCacheData()
		color.Green(" ok!")
		}
		if c.CleanReplays {
		fmt.Print("Starting cleaning useless replays...")
		wg.Add(1)
		go opCleanReplays()
		color.Green(" ok!")
		}
		if c.DeleteReplayCache {
		fmt.Print("Starting deleting replay cache...")
		wg.Add(1)
		go opDeleteReplayCache()
		color.Green(" ok!")
		}

		wg.Wait()
		color.Green("Data elaboration has been terminated.")
		color.Yellow("Waiting for workers to finish...")
		close(execOperations)
		chanWg.Wait()
		}

		// db operation to be made, generally used for execOperations
		type operation struct {
		query string
		params []interface{}
		}

		func op(query string, params ...interface{}) {
		execOperations <- operation{query, params}
		}

		// Operations that can be executed with a simple db.Exec, distributed across 8 workers.
		var execOperations = make(chan operation, 10000)

		func worker() {
		for op := range execOperations {
		if c.LogQueries {
		// porcodio go se sei odioso a volte
		a := []interface{}{
		"=>",
		op.query,
		"\| params:",
		}
		a = append(a, op.params...)
		fmt.Println(a...)
		}
		_, err := db.Exec(op.query, op.params...)
		if err != nil {
		queryError(err, op.query, op.params...)
		}
		}
		chanWg.Done()
		}

		func queryError(err error, query string, params ...interface{}) {
		color.Red(`==> Query error!
		===> %s
		===> params: %v
		===> error: %v`, query, params, err)
		}

replays.go

0 → 100644

+74 −0

Changes for replays.go: 74 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"fmt"
		"io/ioutil"
		"os"

		"github.com/fatih/color"
		)

		func opCleanReplays() {
		dir := removeTrailingSlash(c.RippleDir) + "/osu.ppy.sh/replays"
		if finfo, err := os.Stat(dir); err != nil \|\| !finfo.IsDir() {
		color.Red("> CleanReplays: failed to start cleaning replays:")
		if err != nil {
		color.Red("> %v", err)
		} else {
		color.Red("> %s is a file, not a folder", dir)
		}
		return
		}

		const failedReplays = "SELECT id FROM scores WHERE completed != 3"
		rows, err := db.Query(failedReplays)
		if err != nil {
		queryError(err, failedReplays)
		}
		count := 0
		for rows.Next() {
		if count%50 == 0 && count != 0 {
		fmt.Println("> CleanReplays:", count, "replays cleared")
		}
		var scoreID int
		err := rows.Scan(&scoreID)
		if err != nil {
		queryError(err, failedReplays)
		continue
		}
		filename := fmt.Sprintf("%s/replays/replay_%d.osr", dir, scoreID)
		// We don't check if the file exists, because that would be an useless I/O operation
		// TODO: WorkGroup?
		os.Remove(filename)
		}
		rows.Close()
		wg.Done()
		}
		func removeTrailingSlash(s string) string {
		if s[len(s)-1] == '/' {
		return s[:len(s)-1]
		}
		return s
		}

		func opDeleteReplayCache() {
		dir := removeTrailingSlash(c.RippleDir) + "/osu.ppy.sh/replays_full"
		files, err := ioutil.ReadDir(dir)
		if err != nil {
		color.Red("> DeleteReplaysFull: Couldn't get files from replays_full directory: %v", err)
		return
		}

		count := 0
		for _, file := range files {
		err := os.Remove(dir + "/" + file.Name())
		if err != nil {
		color.Red("> DeleteReplaysFull: couldn't remove file %s: %v", file.Name(), err)
		continue
		}
		count++
		}
		fmt.Println("> DeleteReplaysFull:", count, "replays deleted")

		wg.Done()
		}