-1

.gitignore

reviewed

··· 1 1 .env 2 2 site/ 3 3 - tse/*.zip

-7

.tangled/workflows/deploy.yaml

reviewed

··· 5 5 dependencies: 6 6 nixpkgs: 7 7 - awscli2 8 8 - - curl 9 8 - go 10 9 steps: 11 11 - - name: Download candidates 12 12 - command: curl -Lso tse/consulta_cand_2022.zip https://cdn.tse.jus.br/estatistica/sead/odsele/consulta_cand/consulta_cand_2022.zip 13 13 - - name: Download photos 14 14 - command: curl -Lso tse/foto_cand2022_SC_div.zip https://cdn.tse.jus.br/estatistica/sead/eleicoes/eleicoes2022/fotos/foto_cand2022_SC_div.zip 15 15 - - name: Download social media URLs 16 16 - command: curl -Lso tse/rede_social_candidato_2022_SC.zip https://cdn.tse.jus.br/estatistica/sead/odsele/consulta_cand/rede_social_candidato_2022_SC.zip 17 10 - name: Compile 18 11 command: go build -o desanda main.go 19 12 - name: "Build"

+2 -6

README.md

reviewed

··· 1 1 # 🤬 Assembleia Legislativa Desanda Catarina 2 2 3 3 - 1. Baixar os [dados do TSE de candiaturas para as eleições gerais de 2022](https://dadosabertos.tse.jus.br/dataset/candidatos-2022) e salvar em `tse/`: 4 4 - 1. `consulta_cand_2022.zip` 5 5 - 1. `foto_cand2022_SC_div.zip` 6 6 - 1. `rede_social_candidato_2022_SC.zip` 7 7 - 2. Compilar com, por exemplo, `go build -o /usr/local/bin/desanda` 8 8 - 3. Gerar o site estático em `site/` com `desanda` 3 3 + 1. Compilar com, por exemplo, `go build -o /usr/local/bin/desanda` 4 4 + 2. Gerar o site estático em `site/` com `desanda`

+93 -29

main.go

reviewed

··· 290 290 return "fas fa-globe" 291 291 } 292 292 293 293 - func socialMediaFor(m *map[string][]string, client *http.Client, id string) ([]SocialMedia, error) { 294 294 - var urls []SocialMedia 293 293 + func socialMediaFor(urls []string, client *http.Client, id string) ([]SocialMedia, error) { 295 294 var g errgroup.Group 296 295 ch := make(chan string) 297 297 - for _, url := range (*m)[id] { 296 296 + done := make(chan struct{}) 297 297 + var out []SocialMedia 298 298 + go func() { 299 299 + defer close(done) 300 300 + for url := range ch { 301 301 + out = append(out, SocialMedia{URL: url, Icon: socialIcon(url)}) 302 302 + } 303 303 + }() 304 304 + for _, url := range urls { 298 305 g.Go(func() error { 299 306 resp, err := client.Head(url) 300 307 if err != nil { 301 308 var nf *net.DNSError 302 309 if errors.As(err, &nf) && nf.IsNotFound { 310 310 + return nil 311 311 + } 312 312 + if _, ok := err.(net.Error); ok { 303 313 return nil 304 314 } 305 315 return fmt.Errorf("could not check %s: %w", url, err) ··· 316 326 return nil 317 327 }) 318 328 } 319 319 - go func() { 320 320 - defer close(ch) 321 321 - if err := g.Wait(); err != nil { 322 322 - slog.Warn("error waiting for social media checks", "error", err) 329 329 + err := g.Wait() 330 330 + close(ch) 331 331 + if err != nil { 332 332 + return nil, err 333 333 + } 334 334 + <-done 335 335 + return out, nil 336 336 + } 337 337 + 338 338 + func downloadFile(client *http.Client, url, dir string) error { 339 339 + slog.Info("Downloading", "url", url) 340 340 + defer slog.Info("Finished", "url", url) 341 341 + resp, err := client.Get(url) 342 342 + if err != nil { 343 343 + return err 344 344 + } 345 345 + defer func() { 346 346 + if err := resp.Body.Close(); err != nil { 347 347 + slog.Warn("could not close response body", "url", url, "error", err) 323 348 } 324 349 }() 325 325 - for url := range ch { 326 326 - urls = append(urls, SocialMedia{URL: url, Icon: socialIcon(url)}) 350 350 + if resp.StatusCode != http.StatusOK { 351 351 + return fmt.Errorf("unexpected status for %s: %s", url, resp.Status) 327 352 } 328 328 - return urls, nil 353 353 + pth := filepath.Join(dir, filepath.Base(url)) 354 354 + f, err := os.Create(pth) 355 355 + if err != nil { 356 356 + return err 357 357 + } 358 358 + defer func() { 359 359 + if err := f.Close(); err != nil { 360 360 + slog.Warn("could not close file", "path", pth, "error", err) 361 361 + } 362 362 + }() 363 363 + _, err = io.Copy(f, resp.Body) 364 364 + return err 329 365 } 330 366 331 367 func main() { ··· 333 369 defer func() { 334 370 slog.Info("site built", "elapsed", time.Since(init)) 335 371 }() 372 372 + tmp, err := os.MkdirTemp("", "desanda-*") 373 373 + if err != nil { 374 374 + log.Fatal(err) 375 375 + } 376 376 + defer func() { 377 377 + if err := os.RemoveAll(tmp); err != nil { 378 378 + slog.Warn("could not remove temp dir", "path", tmp, "error", err) 379 379 + } 380 380 + }() 381 381 + client := &http.Client{ 382 382 + CheckRedirect: func(req *http.Request, via []*http.Request) error { 383 383 + return http.ErrUseLastResponse 384 384 + }, 385 385 + Transport: &http.Transport{ 386 386 + MaxConnsPerHost: 8, 387 387 + TLSClientConfig: &tls.Config{ 388 388 + InsecureSkipVerify: true, 389 389 + }, 390 390 + }, 391 391 + } 392 392 + var d errgroup.Group 393 393 + urls := []string{ 394 394 + "https://cdn.tse.jus.br/estatistica/sead/odsele/consulta_cand/consulta_cand_2022.zip", 395 395 + "https://cdn.tse.jus.br/estatistica/sead/eleicoes/eleicoes2022/fotos/foto_cand2022_SC_div.zip", 396 396 + "https://cdn.tse.jus.br/estatistica/sead/odsele/consulta_cand/rede_social_candidato_2022_SC.zip", 397 397 + } 398 398 + for _, u := range urls { 399 399 + d.Go(func() error { return downloadFile(client, u, tmp) }) 400 400 + } 401 401 + if err := d.Wait(); err != nil { 402 402 + log.Fatal(err) 403 403 + } 336 404 slugs, err := readData() 337 405 if err != nil { 338 406 log.Fatal(err) 339 407 } 340 340 - urls, err := csvFromZip("tse/rede_social_candidato_2022_SC.zip", "rede_social_candidato_2022_SC.csv") 408 408 + all, err := csvFromZip(filepath.Join(tmp, "rede_social_candidato_2022_SC.zip"), "rede_social_candidato_2022_SC.csv") 341 409 if err != nil { 342 410 log.Fatal(err) 343 411 } 344 412 m := make(map[string][]string) 345 345 - for i, row := range urls { 413 413 + for i, row := range all { 346 414 if i == 0 { 347 415 continue 348 416 } 349 417 m[row[8]] = append(m[row[8]], row[10]) 350 418 } 351 351 - rows, err := csvFromZip("tse/consulta_cand_2022.zip", "consulta_cand_2022_SC.csv") 419 419 + rows, err := csvFromZip(filepath.Join(tmp, "consulta_cand_2022.zip"), "consulta_cand_2022_SC.csv") 352 420 if err != nil { 353 421 log.Fatal(err) 354 422 } 355 355 - client := &http.Client{ 356 356 - CheckRedirect: func(req *http.Request, via []*http.Request) error { 357 357 - return http.ErrUseLastResponse 358 358 - }, 359 359 - Transport: &http.Transport{ 360 360 - MaxConnsPerHost: 8, 361 361 - TLSClientConfig: &tls.Config{ 362 362 - InsecureSkipVerify: true, 363 363 - }, 364 364 - }, 365 365 - } 366 423 var g errgroup.Group 367 424 g.SetLimit(16) 368 425 idIdx := 15 ··· 376 433 for p := range ch { 377 434 t++ 378 435 ps = append(ps, p) 379 379 - fmt.Printf("%02d. %s\n", t, p.Name) 436 436 + slog.Info("Matched", "count", t, "name", p.Name) 380 437 } 381 438 sort.Slice(ps, func(i, j int) bool { 382 439 return ps[i].Name < ps[j].Name ··· 398 455 if len(v) == 0 { 399 456 return nil 400 457 } 401 401 - pic, err := copyPhotoFrom(id, "tse/foto_cand2022_SC_div.zip") 458 458 + pic, err := copyPhotoFrom(id, filepath.Join(tmp, "foto_cand2022_SC_div.zip")) 402 459 if err != nil { 403 460 return fmt.Errorf("could not get photo for %s (%s): %w", n, id, err) 404 461 } 405 462 if pic == "" { 406 463 return fmt.Errorf("could not get photo for %s (%s)", n, id) 407 464 } 408 408 - sm, err := socialMediaFor(&m, client, id) 409 409 - if err != nil { 410 410 - return fmt.Errorf("could not get social media for %s (%s): %w", n, id, err) 465 465 + var sm []SocialMedia 466 466 + urls, ok := m[id] 467 467 + if ok { 468 468 + sm, err = socialMediaFor(urls, client, id) 469 469 + if err != nil { 470 470 + return fmt.Errorf("could not get social media for %s (%s): %w", n, id, err) 471 471 + } 411 472 } 412 473 p := Person{ 413 474 id, ··· 427 488 } 428 489 close(ch) 429 490 <-done 491 491 + if len(ps) == 0 { 492 492 + log.Fatal("could not find data to populate the site") 493 493 + } 430 494 if err := os.MkdirAll("site", 0755); err != nil { 431 495 log.Fatal(err) 432 496 }

tse/.gitkeep

reviewed

This is a binary file and will not be displayed.