+40

alesc_tse.csv

reviewed

··· 1 1 + ant%C3%ADdio-lunelli,240001605916 2 2 + jair-miotto,240001616203 3 3 + lucas-neves,240001610819 4 4 + m%C3%A1rio-motta,240001614375 5 5 + marquito,240001610066 6 6 + maur%C3%ADcio-peixer,240001614353 7 7 + sargento-lima,240001614329 8 8 + volnei-weber,240001605921 9 9 + altair-silva,240001644767 10 10 + padre-pedro-baldissera,240001679648 11 11 + rodrigo-minotto,240001644398 12 12 + matheus-cadorin,240001597559 13 13 + jess%C3%A9-lopes,240001614350 14 14 + dr.-vicente-caropreso,240001612510 15 15 + neodi-saretta,240001679646 16 16 + oscar-gutz,240001614346 17 17 + pep%C3%AA-colla%C3%A7o,240001644772 18 18 + tiago-zilli,240001605911 19 19 + alex-brasil-,240001614349 20 20 + fabiano-da-luz,240001679627 21 21 + marcos-da-rosa,240001616186 22 22 + maur%C3%ADcio-eskudlark,240001614331 23 23 + paulinha,240001610824 24 24 + sergio-motta,240001610226 25 25 + ana-campagnolo,240001614348 26 26 + marcos-vieira,240001612502 27 27 + nilso-berlanda,240001614335 28 28 + julio-garcia,240001614390 29 29 + marcius-machado,240001614339 30 30 + mauro-de-nadal,240001605913 31 31 + napole%C3%A3o-bernardes-,240001614384 32 32 + luciane-carminatti,240001679643 33 33 + jos%C3%A9-milton-scheffer,240001644756 34 34 + camilo-martins,240001610835 35 35 + emerson-stein,240001605929 36 36 + fernando-krelling,240001605919 37 37 + ivan-naatz,240001614356 38 38 + junior-cardoso,240001645371 39 39 + s%C3%A9rgio-guimar%C3%A3es,240001616196 40 40 + carlos-humberto,240001614337

+160 -145

main.go

reviewed

··· 2 2 3 3 import ( 4 4 "archive/zip" 5 5 + "crypto/tls" 5 6 _ "embed" 6 7 "encoding/csv" 7 8 "errors" ··· 10 11 "io" 11 12 "log" 12 13 "log/slog" 14 14 + "net" 13 15 "net/http" 14 16 "os" 15 17 "path/filepath" 16 18 "sort" 17 19 "strings" 20 20 + "time" 18 21 19 22 "golang.org/x/sync/errgroup" 20 23 "golang.org/x/text/encoding/charmap" ··· 24 27 //go:embed index.html 25 28 var index string 26 29 27 27 - type bill struct { 30 30 + type votes struct { 28 31 name string 29 32 inFavor []string 30 33 against []string 31 34 } 32 35 33 36 var ( 34 34 - quotas = bill{ 35 35 - "Contra cotas raciais na UDESC", 37 37 + quotas = votes{ 38 38 + "Contra cotas raciais", 36 39 []string{ 37 40 "ALEX BRASIL", 38 41 "ALTAIR SILVA", ··· 76 79 "DR. VICENTE", 77 80 }, 78 81 } 79 79 - cameras = bill{ 82 82 + cameras = votes{ 80 83 "Câmeras em sala de aula", 81 84 []string{ 82 85 "PADRE PEDRO", ··· 136 139 type Person struct { 137 140 ID string 138 141 Name string 139 139 - Status string 140 140 - SocialMedia []SocialMedia 141 141 - Photo *string 142 142 + Party string 143 143 + Photo string 142 144 URL string 143 143 - Party string 144 145 Votes []Bill 146 146 + SocialMedia []SocialMedia 145 147 } 146 148 147 147 - func urlFor(client *http.Client, name string) (string, bool, error) { 148 148 - n := strings.ToLower(name) 149 149 - switch { 150 150 - case strings.Contains(n, "marquito"): 151 151 - n = "marquito" 152 152 - case strings.Contains(n, "padre pedro"): 153 153 - n = "padre-pedro-baldissera" 154 154 - case strings.Contains(n, "dr. vicente"): 155 155 - n = "dr.-vicente-caropreso" 156 156 - default: 157 157 - n = strings.ReplaceAll(n, " ", "-") 158 158 - n = strings.ReplaceAll(n, ".", "") 159 159 - n = strings.ReplaceAll(n, "'", "") 160 160 - } 161 161 - url := fmt.Sprintf("https://www.alesc.sc.gov.br/deputados/%s/", n) 162 162 - resp, err := client.Get(url) 149 149 + func readData() (map[string]string, error) { 150 150 + f, err := os.Open("alesc_tse.csv") 163 151 if err != nil { 164 164 - slog.Warn("HTTP error", "error", err, "url", url) 165 165 - return "", false, err 152 152 + return nil, err 166 153 } 167 154 defer func() { 168 168 - if err := resp.Body.Close(); err != nil { 169 169 - slog.Warn("could not close response body", "error", err, "url", url) 155 155 + if err := f.Close(); err != nil { 156 156 + slog.Warn("could not close CSV file", "error", err) 170 157 } 171 158 }() 172 172 - if resp.StatusCode != http.StatusOK { 173 173 - return "", false, errors.New(resp.Status) 159 159 + r := csv.NewReader(f) 160 160 + rows, err := r.ReadAll() 161 161 + if err != nil { 162 162 + return nil, err 174 163 } 175 175 - return url, true, nil 164 164 + m := make(map[string]string) 165 165 + for _, row := range rows { 166 166 + m[row[1]] = row[0] 167 167 + } 168 168 + return m, nil 176 169 } 177 170 178 171 func copyPhotoFrom(id string, pth string) (string, error) { ··· 187 180 }() 188 181 for _, f := range r.File { 189 182 n := filepath.Base(f.Name) 190 190 - if strings.HasPrefix(n, fmt.Sprintf("FSC%s_div", id)) { 191 191 - src, err := f.Open() 192 192 - if err != nil { 193 193 - return "", err 183 183 + if !strings.HasPrefix(n, fmt.Sprintf("FSC%s_div", id)) { 184 184 + continue 185 185 + } 186 186 + src, err := f.Open() 187 187 + if err != nil { 188 188 + return "", err 189 189 + } 190 190 + defer func() { 191 191 + if err := src.Close(); err != nil { 192 192 + slog.Warn("could not close zip file source", "error", err) 194 193 } 195 195 - defer func() { 196 196 - if err := src.Close(); err != nil { 197 197 - slog.Warn("could not close zip file source", "error", err) 198 198 - } 199 199 - }() 200 200 - if err := os.MkdirAll("site/images", 0755); err != nil { 201 201 - return "", err 202 202 - } 203 203 - dst, err := os.Create(fmt.Sprintf("site/images/%s.jpg", id)) 204 204 - if err != nil { 205 205 - return "", err 206 206 - } 207 207 - defer func() { 208 208 - if err := dst.Close(); err != nil { 209 209 - slog.Warn("could not close destination file", "error", err, "id", id) 210 210 - } 211 211 - }() 212 212 - if _, err := io.Copy(dst, src); err != nil { 213 213 - return "", err 194 194 + }() 195 195 + if err := os.MkdirAll("site/images", 0755); err != nil { 196 196 + return "", err 197 197 + } 198 198 + dst, err := os.Create(fmt.Sprintf("site/images/%s.jpg", id)) 199 199 + if err != nil { 200 200 + return "", err 201 201 + } 202 202 + defer func() { 203 203 + if err := dst.Close(); err != nil { 204 204 + slog.Warn("could not close destination file", "error", err, "id", id) 214 205 } 215 215 - return fmt.Sprintf("images/%s.jpg", id), nil 206 206 + }() 207 207 + if _, err := io.Copy(dst, src); err != nil { 208 208 + return "", err 216 209 } 210 210 + return fmt.Sprintf("images/%s.jpg", id), nil 217 211 } 218 212 return "", nil 219 213 } 220 214 221 221 - func getVoteFor(name string, b bill) string { 215 215 + func getVoteFor(name string, b votes) string { 222 216 for _, nm := range b.against { 223 217 if strings.EqualFold(nm, name) { 224 218 return "no" ··· 233 227 } 234 228 235 229 func getBillsFor(name string) []Bill { 236 236 - return []Bill{ 237 237 - {Name: quotas.name, Vote: getVoteFor(name, quotas)}, 238 238 - {Name: cameras.name, Vote: getVoteFor(name, cameras)}, 230 230 + var b []Bill 231 231 + for _, d := range []votes{quotas, cameras} { 232 232 + v := getVoteFor(name, d) 233 233 + if v == "" { 234 234 + continue 235 235 + } 236 236 + b = append(b, Bill{d.name, v}) 239 237 } 238 238 + return b 239 239 + } 240 240 + 241 241 + func csvFromZip(pth string, name string) ([][]string, error) { 242 242 + r, err := zip.OpenReader(pth) 243 243 + if err != nil { 244 244 + return nil, err 245 245 + } 246 246 + defer func() { 247 247 + if err := r.Close(); err != nil { 248 248 + slog.Warn("could not close file", "path", pth, "error", err) 249 249 + } 250 250 + }() 251 251 + for _, f := range r.File { 252 252 + if filepath.Base(f.Name) == name { 253 253 + src, err := f.Open() 254 254 + if err != nil { 255 255 + return nil, err 256 256 + } 257 257 + defer func() { 258 258 + if err := src.Close(); err != nil { 259 259 + slog.Warn("could not close archived file", "path", pth, "name", f.Name, "error", err) 260 260 + } 261 261 + }() 262 262 + r := csv.NewReader(transform.NewReader(src, charmap.Windows1252.NewDecoder())) 263 263 + r.Comma = ';' 264 264 + r.LazyQuotes = true 265 265 + return r.ReadAll() 266 266 + } 267 267 + } 268 268 + return nil, fmt.Errorf("file %s not found in %s", name, pth) 240 269 } 241 270 242 271 func socialIcon(url string) string { ··· 261 290 return "fas fa-globe" 262 291 } 263 292 264 264 - func checkURL(client *http.Client, url string) bool { 265 265 - resp, err := client.Head(url) 266 266 - if err != nil { 267 267 - return false 268 268 - } 269 269 - defer func() { 270 270 - if err := resp.Body.Close(); err != nil { 271 271 - log.Printf("Error closing response body for %s: %v", url, err) 272 272 - } 273 273 - }() 274 274 - return resp.StatusCode == http.StatusOK 275 275 - } 276 276 - 277 277 - func readCSVFromZip(pth string, name string) ([][]string, error) { 278 278 - r, err := zip.OpenReader(pth) 279 279 - if err != nil { 280 280 - return nil, err 281 281 - } 282 282 - defer func() { 283 283 - if err := r.Close(); err != nil { 284 284 - log.Printf("Error closing zip reader for %s: %v", pth, err) 285 285 - } 286 286 - }() 287 287 - for _, f := range r.File { 288 288 - if filepath.Base(f.Name) == name { 289 289 - src, err := f.Open() 293 293 + func socialMediaFor(m *map[string][]string, client *http.Client, id string) ([]SocialMedia, error) { 294 294 + var urls []SocialMedia 295 295 + var g errgroup.Group 296 296 + ch := make(chan string) 297 297 + for _, url := range (*m)[id] { 298 298 + g.Go(func() error { 299 299 + resp, err := client.Head(url) 290 300 if err != nil { 291 291 - return nil, err 301 301 + var nf *net.DNSError 302 302 + if errors.As(err, &nf) && nf.IsNotFound { 303 303 + return nil 304 304 + } 305 305 + return fmt.Errorf("could not check %s: %w", url, err) 292 306 } 293 307 defer func() { 294 294 - if err := src.Close(); err != nil { 295 295 - log.Printf("Error closing zip file source: %v", err) 308 308 + if err := resp.Body.Close(); err != nil { 309 309 + slog.Warn("could not close http response body", "url", url, "error", err) 296 310 } 297 311 }() 298 298 - reader := transform.NewReader(src, charmap.Windows1252.NewDecoder()) 299 299 - r := csv.NewReader(reader) 300 300 - r.Comma = ';' 301 301 - r.LazyQuotes = true 302 302 - return r.ReadAll() 312 312 + ok := resp.StatusCode >= 200 && resp.StatusCode < 400 313 313 + if ok { 314 314 + ch <- url 315 315 + } 316 316 + return nil 317 317 + }) 318 318 + } 319 319 + go func() { 320 320 + defer close(ch) 321 321 + if err := g.Wait(); err != nil { 322 322 + slog.Warn("error waiting for social media checks", "error", err) 303 323 } 324 324 + }() 325 325 + for url := range ch { 326 326 + urls = append(urls, SocialMedia{URL: url, Icon: socialIcon(url)}) 304 327 } 305 305 - return nil, fmt.Errorf("file %s not found in %s", name, pth) 328 328 + return urls, nil 306 329 } 307 330 308 331 func main() { 309 309 - urls, err := readCSVFromZip("tse/rede_social_candidato_2022_SC.zip", "rede_social_candidato_2022_SC.csv") 332 332 + init := time.Now() 333 333 + defer func() { 334 334 + slog.Info("site built", "elapsed", time.Since(init)) 335 335 + }() 336 336 + slugs, err := readData() 337 337 + if err != nil { 338 338 + log.Fatal(err) 339 339 + } 340 340 + urls, err := csvFromZip("tse/rede_social_candidato_2022_SC.zip", "rede_social_candidato_2022_SC.csv") 310 341 if err != nil { 311 342 log.Fatal(err) 312 343 } ··· 315 346 if i == 0 { 316 347 continue 317 348 } 318 318 - id := row[8] 319 319 - url := row[10] 320 320 - m[id] = append(m[id], url) 349 349 + m[row[8]] = append(m[row[8]], row[10]) 321 350 } 322 322 - all, err := readCSVFromZip("tse/consulta_cand_2022.zip", "consulta_cand_2022_SC.csv") 351 351 + rows, err := csvFromZip("tse/consulta_cand_2022.zip", "consulta_cand_2022_SC.csv") 323 352 if err != nil { 324 353 log.Fatal(err) 325 354 } 355 355 + client := &http.Client{ 356 356 + CheckRedirect: func(req *http.Request, via []*http.Request) error { 357 357 + return http.ErrUseLastResponse 358 358 + }, 359 359 + Transport: &http.Transport{ 360 360 + MaxConnsPerHost: 8, 361 361 + TLSClientConfig: &tls.Config{ 362 362 + InsecureSkipVerify: true, 363 363 + }, 364 364 + }, 365 365 + } 326 366 var g errgroup.Group 327 367 g.SetLimit(16) 328 368 idIdx := 15 329 369 nameIdx := 18 330 370 partyIdx := 26 331 331 - statusIdx := 49 332 371 ch := make(chan Person) 333 372 done := make(chan struct{}) 334 334 - client := &http.Client{ 335 335 - CheckRedirect: func(req *http.Request, via []*http.Request) error { 336 336 - return http.ErrUseLastResponse 337 337 - }, 338 338 - } 339 373 var ps []Person 340 374 go func() { 341 375 var t uint ··· 349 383 }) 350 384 close(done) 351 385 }() 352 352 - for i, row := range all { 386 386 + for i, row := range rows { 353 387 if i == 0 { 354 388 continue 355 389 } 356 356 - s := row[statusIdx] 357 357 - if s == "4" || s == "-1" { 358 358 - continue 359 359 - } 360 390 g.Go(func() error { 361 391 id := row[idIdx] 362 362 - p := Person{ID: id, Name: row[nameIdx], Party: row[partyIdx]} 363 363 - p.Name = strings.TrimPrefix(strings.TrimSpace(p.Name), "A ") 364 364 - p.Votes = getBillsFor(p.Name) 365 365 - ok := false 366 366 - for _, b := range p.Votes { 367 367 - if b.Vote != "" { 368 368 - ok = true 369 369 - break 370 370 - } 371 371 - } 392 392 + slug, ok := slugs[id] 372 393 if !ok { 373 394 return nil 374 395 } 375 375 - u, ok, err := urlFor(client, p.Name) 376 376 - if err != nil || !ok { 396 396 + n := strings.TrimPrefix(strings.TrimSpace(row[nameIdx]), "A ") // bad data? 397 397 + v := getBillsFor(n) 398 398 + if len(v) == 0 { 377 399 return nil 378 400 } 379 379 - p.URL = u 380 401 pic, err := copyPhotoFrom(id, "tse/foto_cand2022_SC_div.zip") 381 381 - if err == nil && pic != "" { 382 382 - p.Photo = &pic 402 402 + if err != nil { 403 403 + return fmt.Errorf("could not get photo for %s (%s): %w", n, id, err) 383 404 } 384 384 - var us []SocialMedia 385 385 - var sg errgroup.Group 386 386 - c := make(chan string, len(m[id])) 387 387 - for _, u := range m[id] { 388 388 - u := u 389 389 - sg.Go(func() error { 390 390 - if checkURL(client, u) { 391 391 - c <- u 392 392 - } 393 393 - return nil 394 394 - }) 405 405 + if pic == "" { 406 406 + return fmt.Errorf("could not get photo for %s (%s)", n, id) 395 407 } 396 396 - go func() { 397 397 - if err := sg.Wait(); err != nil { 398 398 - slog.Warn("error waiting for social media checks", "error", err) 399 399 - } 400 400 - close(c) 401 401 - }() 402 402 - for u := range c { 403 403 - us = append(us, SocialMedia{URL: u, Icon: socialIcon(u)}) 408 408 + sm, err := socialMediaFor(&m, client, id) 409 409 + if err != nil { 410 410 + return fmt.Errorf("could not get social media for %s (%s): %w", n, id, err) 411 411 + } 412 412 + p := Person{ 413 413 + id, 414 414 + n, 415 415 + row[partyIdx], 416 416 + pic, 417 417 + fmt.Sprintf("https://www.alesc.sc.gov.br/deputados/%s/", slug), 418 418 + v, 419 419 + sm, 404 420 } 405 405 - p.SocialMedia = us 406 421 ch <- p 407 422 return nil 408 423 })