package services import ( "encoding/csv" "fmt" "log" "strings" "github.com/pkg/errors" "go-sjles-pta-vote/server/db" ) type Member struct { Name string Email string } const BATCH_SIZE = 100 func ParseMembersFromBytes(year int, fileBytes []byte) error { reader := csv.NewReader(strings.NewReader(string(fileBytes))) reader.FieldsPerRecord = -1 // Allow variable number of fields per record records, err := reader.ReadAll() if err != nil { return errors.Wrap(err, "failed to read CSV from bytes") } var members []Member for i, record := range records { if i == 0 { continue // Skip the first line (column headers) } if len(record) < 4 { continue } firstName := strings.TrimSpace(record[1]) lastName := strings.TrimSpace(record[2]) email := strings.TrimSpace(record[3]) members = append(members, Member{ Name: fmt.Sprintf("%s %s", firstName, lastName), Email: email, }) if len(record) < 30 { continue } email2 := strings.TrimSpace(record[27]) if email2 != "" { firstName2 := strings.TrimSpace(record[29]) lastName2 := strings.TrimSpace(record[28]) members = append(members, Member{ Name: fmt.Sprintf("%s %s", firstName2, lastName2), Email: email2, }) } } return saveMember(year, members) } func saveMember(year int, members []Member) error { insertMembersQuery := ` INSERT OR REPLACE INTO members (email, member_name, school_year) VALUES ($1, $2, $3) ` log.Printf("Starting to save %d members for year %d", len(members), year) db_conn, err := db.Connect() if err != nil { return errors.Wrap(err, "failed to connect to database") } defer db_conn.Close() tx, err := db_conn.Begin() if err != nil { return errors.Wrap(err, "failed to begin transaction") } stmt, err := tx.Prepare(insertMembersQuery) if err != nil { tx.Rollback() return errors.Wrap(err, "failed to prepare statement") } defer stmt.Close() for index, member := range members { _, err = stmt.Exec(member.Email, member.Name, year) if err != nil { tx.Rollback() return errors.Wrap(err, "failed to execute insert") } if (index+1) % BATCH_SIZE == 0 { err = tx.Commit() if err != nil { tx.Rollback() return errors.Wrap(err, "failed to commit transaction") } tx, err = db_conn.Begin() if err != nil { return errors.Wrap(err, "failed to begin new transaction") } stmt, err = tx.Prepare(insertMembersQuery) if err != nil { tx.Rollback() return errors.Wrap(err, "failed to prepare new statement") } } } return tx.Commit() }