main.fillRepositoryStatistics - Code Metrics - Inspection of "Test data update" - fedir/ghstat - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( eb237c...dd5abf )

by Fedir

created 2018-03-25 17:49 UTC

main.fillRepositoryStatistics B

↳ Parent: ghstat.go

Complexity

Conditions

Size

Total Lines	41
Code Lines	38

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	4
eloc	38
nop	5
dl	0
loc	41
rs	8.968
c	0
b	0
f	0

// Copyright 2018 Fedir RYKHTIK. All rights reserved.
// Use of this source code is governed by the GNU GPL 3.0
// license that can be found in the LICENSE file.
package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/fedir/ghstat/httpcache"
	"github.com/fedir/ghstat/timing"
)

func main() {
	var (
		clearHTTPCache         = flag.Bool("cc", false, "Clear HTTP cache")
		clearHTTPCacheDryRun   = flag.Bool("ccdr", false, "Clear HTTP cache (dry run)")
		debug                  = flag.Bool("d", false, "Debug mode")
		resultFileSavePath     = flag.String("f", "", "File path where result CSV file will be saved")
		rateLimitCheck         = flag.Bool("l", false, "Rate limit check")
		repositoriesKeysManual = flag.String("r", "", "Repositories keys")
		tmpFolder              = flag.String("t", "test_data", "Clear HTTP cache (dry run)")
		repositoriesKeys       = []string{}
	)
	flag.Parse()
	if *clearHTTPCache || *clearHTTPCacheDryRun {
		clearHTTPCacheFolder(*tmpFolder, *clearHTTPCacheDryRun)
		os.Exit(0)
	}
	if *rateLimitCheck {
		checkAndPrintRateLimit()
		os.Exit(0)
	}
	if *rateLimitCheck {
		checkAndPrintRateLimit()
		os.Exit(0)
	}
	if *repositoriesKeysManual != "" {
		repositoriesKeys = strings.Split(*repositoriesKeysManual, ",")
	} else {
		repositoriesKeys = []string{
			"astaxie/beego",
			"gohugoio/hugo",
			"gin-gonic/gin",
			"labstack/echo",
			"revel/revel",
			"gobuffalo/buffalo",
			"go-chi/chi",
			"kataras/iris",
		}
	}
	csvFilePath := ""
	if *resultFileSavePath != "" {
		csvFilePath = *resultFileSavePath
	} else {
		csvFilePath = "result.csv"
	}
	var ghData = [][]string{}
	headers := []string{
		"Name",
		"URL",
		"Author",
		"Author's followers",
		"Top 10 contributors followers",
		"Created at",
		"Age in days",
		"Total commits",
		"Total additions",
		"Total deletions",
		"Total code changes",
		"Medium commit size",
		"Stargazers",
		"Forks",
		"Contributors",
		"Active forkers, %",
		"Open issues",
		"Total issues",
		"Closed issues, %",
		"Place",
	}
	ghDataColumnIndexes := map[string]int{
		"nameColumn":                       0,
		"authorsFollowersColumn":           3,
		"top10ContributorsFollowersColumn": 4,
		"totalAdditionsColumn":             8,
		"ageColumn":                        6,
		"totalCommitsColumn":               7,
		"totalDeletionsColumn":             9,
		"totalCodeChangesColumn":           10,
		"mediCommitSizeColumn":             11,
		"stargazersColumn":                 12,
		"activeForkersColumn":              15,
		"closedIssuesPercentageColumn":     18,
		"totalPointsColumnIndex":           19,
	}
	wg := &sync.WaitGroup{}
	dataChan := make(chan []string, len(repositoriesKeys))
	for _, rKey := range repositoriesKeys {
		wg.Add(1)
		go fillRepositoryStatistics(rKey, *tmpFolder, *debug, wg, dataChan)
	}
	for _ = range repositoriesKeys {

		ghData = append(ghData, <-dataChan)
	}
	wg.Wait()
	rateGhData(ghData, ghDataColumnIndexes)
	writeCsv(csvFilePath, headers, ghData)
}

func fillRepositoryStatistics(rKey string, tmpFolder string, debug bool, wg *sync.WaitGroup, dataChan chan []string) {
	defer wg.Done()
	repositoryData := getRepositoryStatistics(rKey, tmpFolder, debug)
	authorLogin := getRepositoryCommits(rKey, tmpFolder, debug)
	authorFollowers := 0
	if authorLogin != "" {
		authorFollowers = getUserFollowers(authorLogin, tmpFolder, debug)
	}
	closedIssues := getRepositoryClosedIssues(rKey, tmpFolder, debug)
	topContributorsFollowers, totalContributors := getRepositoryContributors(rKey, tmpFolder, debug)
	activeForkersPercentage := getActiveForkersPercentage(totalContributors, repositoryData.Forks)
	closedIssuesPercentage := getClosedIssuesPercentage(repositoryData.OpenIssues, int(closedIssues))
	contributionStatistics := getContributionStatistics(rKey, tmpFolder, debug)
	ghProjectData := []string{
		repositoryData.Name,
		fmt.Sprintf("https://github.com/%s", repositoryData.FullName),
		fmt.Sprintf("%s", func(a string) string {
			if a == "" {
				a = "[Account removed]"
			}
			return a
		}(authorLogin)),
		fmt.Sprintf("%d", authorFollowers),
		fmt.Sprintf("%d", topContributorsFollowers),
		fmt.Sprintf("%d/%02d", repositoryData.CreatedAt.Year(), repositoryData.CreatedAt.Month()),
		fmt.Sprintf("%d", int(time.Since(repositoryData.CreatedAt).Seconds()/86400)),
		fmt.Sprintf("%d", contributionStatistics.TotalCommits),
		fmt.Sprintf("%d", contributionStatistics.TotalAdditions),
		fmt.Sprintf("%d", contributionStatistics.TotalDeletions),
		fmt.Sprintf("%d", contributionStatistics.TotalCodeChanges),
		fmt.Sprintf("%d", contributionStatistics.MediumCommitSize),
		fmt.Sprintf("%d", repositoryData.Watchers),
		fmt.Sprintf("%d", repositoryData.Forks),
		fmt.Sprintf("%d", totalContributors),
		fmt.Sprintf("%.2f", activeForkersPercentage),
		fmt.Sprintf("%d", repositoryData.OpenIssues),
		fmt.Sprintf("%d", closedIssues+repositoryData.OpenIssues),
		fmt.Sprintf("%.2f", closedIssuesPercentage),
		"0",
	}
	dataChan <- ghProjectData
}

func clearHTTPCacheFolder(tmpFolderPath string, dryRun bool) error {
	d, err := os.Open(tmpFolderPath)
	if err != nil {
		log.Fatalf("Could not open %s", tmpFolderPath)
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		log.Fatalf("Could not read from %s", tmpFolderPath)
	}
	for _, name := range names {
		fp := filepath.Join(tmpFolderPath, name)
		if dryRun {
			fmt.Printf("Deleted %s\n", fp)
		} else {
			err = os.RemoveAll(fp)
			if err != nil {
				log.Fatalf("Could not remove %s", fp)
			}
			fmt.Printf("Deleted %s\n", fp)
		}
	}
	return nil
}

func checkAndPrintRateLimit() {
	type RateLimits struct {
		Resources struct {
			Core struct {
				Limit     int `json:"limit"`
				Remaining int `json:"remaining"`
				Reset     int `json:"reset"`
			} `json:"core"`
			Search struct {
				Limit     int `json:"limit"`
				Remaining int `json:"remaining"`
				Reset     int `json:"reset"`
			} `json:"search"`
			GraphQL struct {
				Limit     int `json:"limit"`
				Remaining int `json:"remaining"`
				Reset     int `json:"reset"`
			} `json:"graphql"`
		} `json:"resources"`
		Rate struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"rate"`
	}
	url := "https://api.github.com/rate_limit"
	resp, statusCode, err := httpcache.MakeHTTPRequest(url)
	if err != nil {
		log.Fatalf("Error during checking rate limit : %d %v#", statusCode, err)
	}
	jsonResponse, _, _ := httpcache.ReadResp(resp)
	rateLimits := RateLimits{}
	json.Unmarshal(jsonResponse, &rateLimits)
	fmt.Printf("Core: %d/%d (reset in %d minutes)\n", rateLimits.Resources.Core.Remaining, rateLimits.Resources.Core.Limit, timing.GetRelativeTime(rateLimits.Resources.Core.Reset))
	fmt.Printf("Search: %d/%d (reset in %d minutes)\n", rateLimits.Resources.Search.Remaining, rateLimits.Resources.Search.Limit, timing.GetRelativeTime(rateLimits.Resources.Search.Reset))
	fmt.Printf("GraphQL: %d/%d (reset in %d minutes)\n", rateLimits.Resources.GraphQL.Remaining, rateLimits.Resources.GraphQL.Limit, timing.GetRelativeTime(rateLimits.Resources.GraphQL.Reset))
	fmt.Printf("Rate: %d/%d (reset in %d minutes)\n", rateLimits.Rate.Remaining, rateLimits.Rate.Limit, timing.GetRelativeTime(rateLimits.Rate.Reset))
}

func writeCsv(csvFilePath string, headers []string, ghData [][]string) {
	file, err := os.Create(csvFilePath)
	if err != nil {
		log.Fatal("Cannot create file", err)
	}
	defer file.Close()
	writer := csv.NewWriter(file)
	defer writer.Flush()
	err = writer.Write(headers)
	if err != nil {
		log.Fatal("Cannot write to file", err)
	}
	for _, value := range ghData {
		err := writer.Write(value)
		if err != nil {
			log.Fatal("Cannot write to file", err)
		}
	}
}


1			// Copyright 2018 Fedir RYKHTIK. All rights reserved.
2			// Use of this source code is governed by the GNU GPL 3.0
3			// license that can be found in the LICENSE file.
4			package main
5
6			import (
7			"encoding/csv"
8			"encoding/json"
9			"flag"
10			"fmt"
11			"log"
12			"os"
13			"path/filepath"
14			"strings"
15			"sync"
16			"time"
17
18			"github.com/fedir/ghstat/httpcache"
19			"github.com/fedir/ghstat/timing"
20			)
21
22			func main() {
23			var (
24			clearHTTPCache = flag.Bool("cc", false, "Clear HTTP cache")
25			clearHTTPCacheDryRun = flag.Bool("ccdr", false, "Clear HTTP cache (dry run)")
26			debug = flag.Bool("d", false, "Debug mode")
27			resultFileSavePath = flag.String("f", "", "File path where result CSV file will be saved")
28			rateLimitCheck = flag.Bool("l", false, "Rate limit check")
29			repositoriesKeysManual = flag.String("r", "", "Repositories keys")
30			tmpFolder = flag.String("t", "test_data", "Clear HTTP cache (dry run)")
31			repositoriesKeys = []string{}
32			)
33			flag.Parse()
34			if clearHTTPCache \|\| clearHTTPCacheDryRun {
35			clearHTTPCacheFolder(tmpFolder, clearHTTPCacheDryRun)
36			os.Exit(0)
37			}
38			if *rateLimitCheck {
39			checkAndPrintRateLimit()
40			os.Exit(0)
41			}
42			if *rateLimitCheck {
43			checkAndPrintRateLimit()
44			os.Exit(0)
45			}
46			if *repositoriesKeysManual != "" {
47			repositoriesKeys = strings.Split(*repositoriesKeysManual, ",")
48			} else {
49			repositoriesKeys = []string{
50			"astaxie/beego",
51			"gohugoio/hugo",
52			"gin-gonic/gin",
53			"labstack/echo",
54			"revel/revel",
55			"gobuffalo/buffalo",
56			"go-chi/chi",
57			"kataras/iris",
58			}
59			}
60			csvFilePath := ""
61			if *resultFileSavePath != "" {
62			csvFilePath = *resultFileSavePath
63			} else {
64			csvFilePath = "result.csv"
65			}
66			var ghData = [][]string{}
67			headers := []string{
68			"Name",
69			"URL",
70			"Author",
71			"Author's followers",
72			"Top 10 contributors followers",
73			"Created at",
74			"Age in days",
75			"Total commits",
76			"Total additions",
77			"Total deletions",
78			"Total code changes",
79			"Medium commit size",
80			"Stargazers",
81			"Forks",
82			"Contributors",
83			"Active forkers, %",
84			"Open issues",
85			"Total issues",
86			"Closed issues, %",
87			"Place",
88			}
89			ghDataColumnIndexes := map[string]int{
90			"nameColumn": 0,
91			"authorsFollowersColumn": 3,
92			"top10ContributorsFollowersColumn": 4,
93			"totalAdditionsColumn": 8,
94			"ageColumn": 6,
95			"totalCommitsColumn": 7,
96			"totalDeletionsColumn": 9,
97			"totalCodeChangesColumn": 10,
98			"mediCommitSizeColumn": 11,
99			"stargazersColumn": 12,
100			"activeForkersColumn": 15,
101			"closedIssuesPercentageColumn": 18,
102			"totalPointsColumnIndex": 19,
103			}
104			wg := &sync.WaitGroup{}
105			dataChan := make(chan []string, len(repositoriesKeys))
106			for _, rKey := range repositoriesKeys {
107			wg.Add(1)
108			go fillRepositoryStatistics(rKey, tmpFolder, debug, wg, dataChan)
109			}
110			for _ = range repositoriesKeys {
			0 ignored issues – show introduced 2018-03-25 17:51 UTC by Report Bug Copy Issue Report should omit values from range; this loop is equivalent to `for range ...` Loading history...
111			ghData = append(ghData, <-dataChan)
112			}
113			wg.Wait()
114			rateGhData(ghData, ghDataColumnIndexes)
115			writeCsv(csvFilePath, headers, ghData)
116			}
117
118			func fillRepositoryStatistics(rKey string, tmpFolder string, debug bool, wg *sync.WaitGroup, dataChan chan []string) {
119			defer wg.Done()
120			repositoryData := getRepositoryStatistics(rKey, tmpFolder, debug)
121			authorLogin := getRepositoryCommits(rKey, tmpFolder, debug)
122			authorFollowers := 0
123			if authorLogin != "" {
124			authorFollowers = getUserFollowers(authorLogin, tmpFolder, debug)
125			}
126			closedIssues := getRepositoryClosedIssues(rKey, tmpFolder, debug)
127			topContributorsFollowers, totalContributors := getRepositoryContributors(rKey, tmpFolder, debug)
128			activeForkersPercentage := getActiveForkersPercentage(totalContributors, repositoryData.Forks)
129			closedIssuesPercentage := getClosedIssuesPercentage(repositoryData.OpenIssues, int(closedIssues))
130			contributionStatistics := getContributionStatistics(rKey, tmpFolder, debug)
131			ghProjectData := []string{
132			repositoryData.Name,
133			fmt.Sprintf("https://github.com/%s", repositoryData.FullName),
134			fmt.Sprintf("%s", func(a string) string {
135			if a == "" {
136			a = "[Account removed]"
137			}
138			return a
139			}(authorLogin)),
140			fmt.Sprintf("%d", authorFollowers),
141			fmt.Sprintf("%d", topContributorsFollowers),
142			fmt.Sprintf("%d/%02d", repositoryData.CreatedAt.Year(), repositoryData.CreatedAt.Month()),
143			fmt.Sprintf("%d", int(time.Since(repositoryData.CreatedAt).Seconds()/86400)),
144			fmt.Sprintf("%d", contributionStatistics.TotalCommits),
145			fmt.Sprintf("%d", contributionStatistics.TotalAdditions),
146			fmt.Sprintf("%d", contributionStatistics.TotalDeletions),
147			fmt.Sprintf("%d", contributionStatistics.TotalCodeChanges),
148			fmt.Sprintf("%d", contributionStatistics.MediumCommitSize),
149			fmt.Sprintf("%d", repositoryData.Watchers),
150			fmt.Sprintf("%d", repositoryData.Forks),
151			fmt.Sprintf("%d", totalContributors),
152			fmt.Sprintf("%.2f", activeForkersPercentage),
153			fmt.Sprintf("%d", repositoryData.OpenIssues),
154			fmt.Sprintf("%d", closedIssues+repositoryData.OpenIssues),
155			fmt.Sprintf("%.2f", closedIssuesPercentage),
156			"0",
157			}
158			dataChan <- ghProjectData
159			}
160
161			func clearHTTPCacheFolder(tmpFolderPath string, dryRun bool) error {
162			d, err := os.Open(tmpFolderPath)
163			if err != nil {
164			log.Fatalf("Could not open %s", tmpFolderPath)
165			}
166			defer d.Close()
167			names, err := d.Readdirnames(-1)
168			if err != nil {
169			log.Fatalf("Could not read from %s", tmpFolderPath)
170			}
171			for _, name := range names {
172			fp := filepath.Join(tmpFolderPath, name)
173			if dryRun {
174			fmt.Printf("Deleted %s\n", fp)
175			} else {
176			err = os.RemoveAll(fp)
177			if err != nil {
178			log.Fatalf("Could not remove %s", fp)
179			}
180			fmt.Printf("Deleted %s\n", fp)
181			}
182			}
183			return nil
184			}
185
186			func checkAndPrintRateLimit() {
187			type RateLimits struct {
188			Resources struct {
189			Core struct {
190			Limit int `json:"limit"`
191			Remaining int `json:"remaining"`
192			Reset int `json:"reset"`
193			} `json:"core"`
194			Search struct {
195			Limit int `json:"limit"`
196			Remaining int `json:"remaining"`
197			Reset int `json:"reset"`
198			} `json:"search"`
199			GraphQL struct {
200			Limit int `json:"limit"`
201			Remaining int `json:"remaining"`
202			Reset int `json:"reset"`
203			} `json:"graphql"`
204			} `json:"resources"`
205			Rate struct {
206			Limit int `json:"limit"`
207			Remaining int `json:"remaining"`
208			Reset int `json:"reset"`
209			} `json:"rate"`
210			}
211			url := "https://api.github.com/rate_limit"
212			resp, statusCode, err := httpcache.MakeHTTPRequest(url)
213			if err != nil {
214			log.Fatalf("Error during checking rate limit : %d %v#", statusCode, err)
215			}
216			jsonResponse, _, _ := httpcache.ReadResp(resp)
217			rateLimits := RateLimits{}
218			json.Unmarshal(jsonResponse, &rateLimits)
219			fmt.Printf("Core: %d/%d (reset in %d minutes)\n", rateLimits.Resources.Core.Remaining, rateLimits.Resources.Core.Limit, timing.GetRelativeTime(rateLimits.Resources.Core.Reset))
220			fmt.Printf("Search: %d/%d (reset in %d minutes)\n", rateLimits.Resources.Search.Remaining, rateLimits.Resources.Search.Limit, timing.GetRelativeTime(rateLimits.Resources.Search.Reset))
221			fmt.Printf("GraphQL: %d/%d (reset in %d minutes)\n", rateLimits.Resources.GraphQL.Remaining, rateLimits.Resources.GraphQL.Limit, timing.GetRelativeTime(rateLimits.Resources.GraphQL.Reset))
222			fmt.Printf("Rate: %d/%d (reset in %d minutes)\n", rateLimits.Rate.Remaining, rateLimits.Rate.Limit, timing.GetRelativeTime(rateLimits.Rate.Reset))
223			}
224
225			func writeCsv(csvFilePath string, headers []string, ghData [][]string) {
226			file, err := os.Create(csvFilePath)
227			if err != nil {
228			log.Fatal("Cannot create file", err)
229			}
230			defer file.Close()
231			writer := csv.NewWriter(file)
232			defer writer.Flush()
233			err = writer.Write(headers)
234			if err != nil {
235			log.Fatal("Cannot write to file", err)
236			}
237			for _, value := range ghData {
238			err := writer.Write(value)
239			if err != nil {
240			log.Fatal("Cannot write to file", err)
241			}
242			}
243			}
244

fedir / ghstat

GitHub Access Token became invalid

Push — master ( eb237c...dd5abf )

main.fillRepositoryStatistics B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like