repo_list() - Code Metrics - Inspection of "feat : repo extractor added" - sepandhaghighi/gitfollow - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( a89782...a0e972 )

by Sepand

created 2017-06-07 21:38 UTC

repo_list() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	1

Metric	Value
cc	3
c	1
b	0
f	1
dl	0
loc	18
rs	9.4285

import requests
import sys
import socket
import os
import datetime
from functools import reduce
import time
from random import randint
DEBUG=False
import gc


def zero_insert(input_string):
    '''
    This function get a string as input if input is one digit add a zero
    :param input_string: input digit az string
    :type input_string:str
    :return: modified output as str
    '''
    if len(input_string)==1:
        return "0"+input_string
    return input_string

def time_convert(input_string):
    '''
    This function convert input_string from uptime from sec to DD,HH,MM,SS Format
    :param input_string: input time string  in sec
    :type input_string:str
    :return: converted time as string
    '''
    input_sec=float(input_string)
    input_minute=input_sec//60
    input_sec=int(input_sec-input_minute*60)
    input_hour=input_minute//60
    input_minute=int(input_minute-input_hour*60)
    input_day=int(input_hour//24)
    input_hour=int(input_hour-input_day*24)
    return zero_insert(str(input_day))+" days, "+zero_insert(str(input_hour))+" hour, "+zero_insert(str(input_minute))+" minutes, "+zero_insert(str(input_sec))+" seconds"

def url_maker_following(Name,page_number):
    '''
    This function return github following page url
    :param Name: Username
    :param page_number: page nubmer of following page
    :type Name:str
    :type Page:int
    :return: github following url as string
    '''
    return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=following"

def url_maker_repo(Name,page_number):
    '''
    This function return github following page url
    :param Name: Username
    :param page_number: page nubmer of following page
    :type Name:str
    :type Page:int
    :return: github following url as string
    '''
    return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=repositories"
def url_maker_follower(Name,page_number):
    '''

    This function return github follower page url
    :param Name: username
    :param page_number: page number of follower page
    :type Name:str
    :type page_number:int
    :return: github follower url as string
    '''
    return "https://github.com/" + Name + "?page=" + str(page_number) + "&tab=followers"
def url_maker_star(Name,page_number):
    '''
    This function return github stars page url
    :param Name: username
    :param page_number: page number of stars
    :type Name :str
    :type page_number:int
    :return: github star url as string
    '''
    return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=stars"

def repo_extract(input_string,username):
    '''
    This function extract repo from raw_html
    :param input_string: raw input html
    :param follower_name: follower_name
    :type input_string:str
    :type follower_name:str
    :return: repo_list as list
    '''
    user_list=[]
    index=0
    shift=len(username)+1
    while(index!=-1):
        index=input_string.find('src="/'+username,index+shift,len(input_string))
        length=input_string[index:].find('graphs/')
        star_repo=input_string[index+5:index+length]
        if star_repo.find("<svg")==-1 and len(star_repo)!=0:
            user_list.append(star_repo)
    return user_list
def star_extract(input_string):
    '''
    This function extract stared repo from raw_html
    :param input_string: raw input html
    :param follower_name: follower_name
    :type input_string:str
    :type follower_name:str
    :return: user_list as list
    '''
    user_list=[]
    index=0
    while(index!=-1):
        index=input_string.find('<a class="muted-link mr-3',index+33,len(input_string))
        length=input_string[index+33:].find('stargazers">\n')
        star_repo=input_string[index+34:index+33+length]
        if star_repo.find("<svg")==-1 and len(star_repo)!=0:
            user_list.append(star_repo)
    return user_list

def user_list_gen(input_string,follower_name):
    '''
    This function extract usernames from raw_html
    :param input_string: raw input html
    :param follower_name: follower_name
    :type input_string:str
    :type follower_name:str
    :return: user_list as list
    '''
    user_list=[]
    index=0
    while(index!=-1):
        index=input_string.find('alt="@',index+6,len(input_string))
        length=input_string[index+6:].find('"')
        user_name=input_string[index+6:index+6+length]
        if user_name!=follower_name:
            if user_name!=follower_name:
                user_list.append(user_name)
    return user_list[:-1]


def get_html(url):
    '''
    This function extract raw_html file
    :param url: url
    :type url:str
    :return: html data
    '''
    time.sleep(create_random_sleep())
    if internet()==True:
        new_session=requests.session()
        new_session.cookies.clear()
        raw_html=new_session.get(url)
        new_session.close()
        raw_data=raw_html.text
        if "Not Found" in raw_data:
            print("Invalid Github User")
            sys.exit()
        return raw_data
    else:
        print("Error In Internet")
        sys.exit()


def end_check(input_string):
    '''
    This function check end page
    :param input_string: raw html
    :type input_string:str
    :return: True or False
    '''
    if input_string.find("reached the end")!=-1:
        return True
    else:
        return False
def follower_list_gen(follower_name):
    '''
    This function generate follower_list
    :param follower_name: username
    :type follower_name:str
    :return: username follower list
    '''
    follower_list = []
    page_number=0
    while (True):
        page_number += 1
        follower_url = url_maker_follower(follower_name, page_number)
        follower_html = get_html(follower_url)
        if end_check(follower_html) == True:
            break
        temp_list = user_list_gen(follower_html,follower_name)
        follower_list.extend(temp_list)
    return follower_list
def repo_list(username):
    '''
    This function return stared_repo list
    :param username: username
    :type username:str
    :return: stared repo as list
    '''
    repo_list_temp=[]
    page_number=0
    while (True):
        page_number += 1
        repo_url = url_maker_repo(username, page_number)
        repo_html = get_html(repo_url)
        temp_list = repo_extract(repo_html,username)
        if len(temp_list)==0:
            break
        repo_list_temp.extend(temp_list)
    return repo_list_temp
def star_list(username):
    '''
    This function return stared_repo list
    :param username: username
    :type username:str
    :return: stared repo as list
    '''
    star_list=[]
    page_number=0
    while (True):
        page_number += 1
        star_url = url_maker_star(username, page_number)
        star_html = get_html(star_url)
        temp_list = star_extract(star_html)
        if len(temp_list)==0:
            break
        star_list.extend(temp_list)
    return star_list

def following_list_gen(follower_name):
    '''
    This function generate following list
    :param follower_name: username
    :type follower_name:str
    :return: username following list
    '''
    following_list = []
    page_number=0
    while (True):
        page_number+=1
        following_url = url_maker_following(follower_name, page_number)
        following_html = get_html(following_url)
        if end_check(following_html) == True:
            break
        temp_list = user_list_gen(following_html,follower_name)
        following_list.extend(temp_list)
    return following_list

def error_log(msg):
    """
    Create the errorlog of the app
    :param msg: error message
    :type msg:str
    """
    if "log" not in os.listdir():
        os.mkdir("log")
    file = open(reduce(os.path.join, [os.getcwd(), "log", "error_log.txt"]), "a")
    file.write(str(datetime.datetime.now()) + " --> " + str(msg) + "\n")
    file.close()

def internet(host="8.8.8.8", port=53, timeout=3):
    """
    Check Internet Connections.
    :param  host: the host that check connection to
    :param  port: port that check connection with
    :param  timeout: times that check the connnection
    :type host:str
    :type port:int
    :type timeout:int
    :return bool: True if Connection is Stable
    >>> internet() # if there is stable internet connection
    True
    >>> internet() # if there is no stable internet connection
    False
    """
    try:
        socket.setdefaulttimeout(timeout)
        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
        return True
    except Exception as ex:
        error_log(str(ex))
        return False

def create_random_sleep(index=1,min_time=1,max_time=3):
    '''
    This function generate sleep time with random processes
    :param index: index to determine first page  and messages(index = 0 is for first page)
    :param min_time: minimum time of sleep
    :param max_time: maximum time of sleep
    :type index:int
    :type min_time:int
    :type max_time:int
    :return: time of sleep as integer (a number between max and min)
    '''
    if index==0:
        time_sleep = 5
        if DEBUG==True:
            print("Wait "+str(time_sleep)+" sec for first search . . .")
    else:
        time_sleep = randint(min_time, max_time)
        if DEBUG==True:
            print("Wait "+str(time_sleep)+" sec for next search . . .")
    if DEBUG==True:
        print_line(70,"*")
    return time_sleep

def print_line(number=30,char="-"):
    '''
    This function print line in screen
    :param number: number of items in each line
    :param char: each char of line
    :return: None
    '''
    line=""
    for i in range(number):
        line=line+char
    print(line)


def follow(username):
    '''
    This function create following and follower list
    :param username: username
    :type username:str
    :return: (list_1,list_2) as tuple
    '''
    print("Collecting Follower Information ...")
    print_line(70, "*")
    list_1 = follower_list_gen(username)
    file = open(username + "_follower.log", "w")
    print(str(len(list_1)) + " Followers --> " + username + "_follower.log")
    print_line(70, "*")
    file.write("\n".join(list_1))
    file.close()
    print('Collecting Following Informnation ...')
    print_line(70, "*")
    list_2 = following_list_gen(username)
    file = open(username + "_following.log", "w")
    print(str(len(list_2)) + " Following --> " + username + "_following.log")
    print_line(70, "*")
    file.write("\n".join(list_2))
    file.close()
    print('Collecting Stars Informnation ...')
    print_line(70, "*")
    stars=star_list(username)
    file = open(username + "_stars.log", "w")
    print(str(len(stars)) + " Stars --> " + username + "_stars.log")
    print_line(70, "*")
    file.write("\n".join(stars))
    file.close()

    print('Collecting Repos Informnation ...')
    print_line(70, "*")
    repos = repo_list(username)
    file = open(username + "_repos.log", "w")
    print(str(len(repos)) + " Repos --> " + username + "_repos.log")
    print_line(70, "*")
    file.write("\n".join(repos))
    file.close()
    return (list_1,list_2)

def dif(list_1,list_2):
    '''
    This function generate dif files
    :param list_1:follower list
    :param list_2: following list
    :type list_1:list
    :type list_2:list
    :return: None
    '''
    file = open(username + "_dif1.log", "w")
    dif_list = list(set(list_2) - set(list_1))
    print(str(len(dif_list)) + " Following - Not Follower --> " + username + "_dif1.log")
    print_line(70, "*")
    file.write("\n".join(dif_list))
    file.close()
    file = open(username + "_dif2.log", "w")
    dif_list = list(set(list_1) - set(list_2))
    print(str(len(dif_list)) + " Follower - Not Following --> " + username + "_dif2.log")
    print_line(70, "*")
    file.write("\n".join(dif_list))
    file.close()
if __name__=="__main__":
    time_1=time.perf_counter()
    username=input("Please Enter Your Github Username : ")
    (list_1,list_2)=follow(username)
    dif(list_1,list_2)
    time_2=time.perf_counter()
    dif_time=str(time_2-time_1)
    print("Data Generated In "+time_convert(dif_time)+" sec")
    print("Log Files Are Ready --> " + os.getcwd())
    gc.collect()














1		import requests
2		import sys
3		import socket
4		import os
5		import datetime
6		from functools import reduce
7		import time
8		from random import randint
9		DEBUG=False
10		import gc
11
12
13		def zero_insert(input_string):
14		'''
15		This function get a string as input if input is one digit add a zero
16		:param input_string: input digit az string
17		:type input_string:str
18		:return: modified output as str
19		'''
20		if len(input_string)==1:
21		return "0"+input_string
22		return input_string
23
24		def time_convert(input_string):
25		'''
26		This function convert input_string from uptime from sec to DD,HH,MM,SS Format
27		:param input_string: input time string in sec
28		:type input_string:str
29		:return: converted time as string
30		'''
31		input_sec=float(input_string)
32		input_minute=input_sec//60
33		input_sec=int(input_sec-input_minute*60)
34		input_hour=input_minute//60
35		input_minute=int(input_minute-input_hour*60)
36		input_day=int(input_hour//24)
37		input_hour=int(input_hour-input_day*24)
38		return zero_insert(str(input_day))+" days, "+zero_insert(str(input_hour))+" hour, "+zero_insert(str(input_minute))+" minutes, "+zero_insert(str(input_sec))+" seconds"
39
40		def url_maker_following(Name,page_number):
41		'''
42		This function return github following page url
43		:param Name: Username
44		:param page_number: page nubmer of following page
45		:type Name:str
46		:type Page:int
47		:return: github following url as string
48		'''
49		return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=following"
50
51		def url_maker_repo(Name,page_number):
52		'''
53		This function return github following page url
54		:param Name: Username
55		:param page_number: page nubmer of following page
56		:type Name:str
57		:type Page:int
58		:return: github following url as string
59		'''
60		return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=repositories"
61		def url_maker_follower(Name,page_number):
62	View Code Duplication	'''
		0 ignored issues – show Duplication introduced 2017-06-07 21:39 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
63		This function return github follower page url
64		:param Name: username
65		:param page_number: page number of follower page
66		:type Name:str
67		:type page_number:int
68		:return: github follower url as string
69		'''
70		return "https://github.com/" + Name + "?page=" + str(page_number) + "&tab=followers"
71		def url_maker_star(Name,page_number):
72		'''
73		This function return github stars page url
74		:param Name: username
75		:param page_number: page number of stars
76		:type Name :str
77		:type page_number:int
78		:return: github star url as string
79		'''
80	View Code Duplication	return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=stars"
		0 ignored issues – show Duplication introduced 2017-06-07 21:39 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
81		def repo_extract(input_string,username):
82		'''
83		This function extract repo from raw_html
84		:param input_string: raw input html
85		:param follower_name: follower_name
86		:type input_string:str
87		:type follower_name:str
88		:return: repo_list as list
89		'''
90		user_list=[]
91		index=0
92		shift=len(username)+1
93		while(index!=-1):
94		index=input_string.find('src="/'+username,index+shift,len(input_string))
95		length=input_string[index:].find('graphs/')
96		star_repo=input_string[index+5:index+length]
97		if star_repo.find("<svg")==-1 and len(star_repo)!=0:
98		user_list.append(star_repo)
99		return user_list
100		def star_extract(input_string):
101		'''
102		This function extract stared repo from raw_html
103		:param input_string: raw input html
104		:param follower_name: follower_name
105		:type input_string:str
106		:type follower_name:str
107		:return: user_list as list
108		'''
109		user_list=[]
110		index=0
111		while(index!=-1):
112		index=input_string.find('<a class="muted-link mr-3',index+33,len(input_string))
113		length=input_string[index+33:].find('stargazers">\n')
114		star_repo=input_string[index+34:index+33+length]
115		if star_repo.find("<svg")==-1 and len(star_repo)!=0:
116		user_list.append(star_repo)
117		return user_list
118
119		def user_list_gen(input_string,follower_name):
120		'''
121		This function extract usernames from raw_html
122		:param input_string: raw input html
123		:param follower_name: follower_name
124		:type input_string:str
125		:type follower_name:str
126		:return: user_list as list
127		'''
128		user_list=[]
129		index=0
130		while(index!=-1):
131		index=input_string.find('alt="@',index+6,len(input_string))
132		length=input_string[index+6:].find('"')
133		user_name=input_string[index+6:index+6+length]
134		if user_name!=follower_name:
135		if user_name!=follower_name:
136		user_list.append(user_name)
137		return user_list[:-1]
138
139
140		def get_html(url):
141		'''
142		This function extract raw_html file
143		:param url: url
144		:type url:str
145		:return: html data
146		'''
147		time.sleep(create_random_sleep())
148		if internet()==True:
149		new_session=requests.session()
150		new_session.cookies.clear()
151		raw_html=new_session.get(url)
152		new_session.close()
153		raw_data=raw_html.text
154		if "Not Found" in raw_data:
155		print("Invalid Github User")
156		sys.exit()
157		return raw_data
158		else:
159		print("Error In Internet")
160		sys.exit()
161
162
163		def end_check(input_string):
164		'''
165		This function check end page
166		:param input_string: raw html
167		:type input_string:str
168		:return: True or False
169		'''
170		if input_string.find("reached the end")!=-1:
171		return True
172		else:
173		return False
174		def follower_list_gen(follower_name):
175		'''
176		This function generate follower_list
177		:param follower_name: username
178		:type follower_name:str
179		:return: username follower list
180		'''
181		follower_list = []
182		page_number=0
183		while (True):
184		page_number += 1
185		follower_url = url_maker_follower(follower_name, page_number)
186		follower_html = get_html(follower_url)
187		if end_check(follower_html) == True:
188		break
189		temp_list = user_list_gen(follower_html,follower_name)
190		follower_list.extend(temp_list)
191		return follower_list
192		def repo_list(username):
193		'''
194		This function return stared_repo list
195		:param username: username
196		:type username:str
197		:return: stared repo as list
198		'''
199		repo_list_temp=[]
200		page_number=0
201		while (True):
202		page_number += 1
203		repo_url = url_maker_repo(username, page_number)
204		repo_html = get_html(repo_url)
205		temp_list = repo_extract(repo_html,username)
206		if len(temp_list)==0:
207		break
208		repo_list_temp.extend(temp_list)
209		return repo_list_temp
210		def star_list(username):
211		'''
212		This function return stared_repo list
213		:param username: username
214		:type username:str
215		:return: stared repo as list
216		'''
217		star_list=[]
218		page_number=0
219		while (True):
220		page_number += 1
221		star_url = url_maker_star(username, page_number)
222		star_html = get_html(star_url)
223		temp_list = star_extract(star_html)
224		if len(temp_list)==0:
225		break
226		star_list.extend(temp_list)
227		return star_list
228
229		def following_list_gen(follower_name):
230		'''
231		This function generate following list
232		:param follower_name: username
233		:type follower_name:str
234		:return: username following list
235		'''
236		following_list = []
237		page_number=0
238		while (True):
239		page_number+=1
240		following_url = url_maker_following(follower_name, page_number)
241		following_html = get_html(following_url)
242		if end_check(following_html) == True:
243		break
244		temp_list = user_list_gen(following_html,follower_name)
245		following_list.extend(temp_list)
246		return following_list
247
248		def error_log(msg):
249		"""
250		Create the errorlog of the app
251		:param msg: error message
252		:type msg:str
253		"""
254		if "log" not in os.listdir():
255		os.mkdir("log")
256		file = open(reduce(os.path.join, [os.getcwd(), "log", "error_log.txt"]), "a")
257		file.write(str(datetime.datetime.now()) + " --> " + str(msg) + "\n")
258		file.close()
259
260		def internet(host="8.8.8.8", port=53, timeout=3):
261		"""
262		Check Internet Connections.
263		:param host: the host that check connection to
264		:param port: port that check connection with
265		:param timeout: times that check the connnection
266		:type host:str
267		:type port:int
268		:type timeout:int
269		:return bool: True if Connection is Stable
270		>>> internet() # if there is stable internet connection
271		True
272		>>> internet() # if there is no stable internet connection
273		False
274		"""
275		try:
276		socket.setdefaulttimeout(timeout)
277		socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
278		return True
279		except Exception as ex:
280		error_log(str(ex))
281		return False
282
283		def create_random_sleep(index=1,min_time=1,max_time=3):
284		'''
285		This function generate sleep time with random processes
286		:param index: index to determine first page and messages(index = 0 is for first page)
287		:param min_time: minimum time of sleep
288		:param max_time: maximum time of sleep
289		:type index:int
290		:type min_time:int
291		:type max_time:int
292		:return: time of sleep as integer (a number between max and min)
293		'''
294		if index==0:
295		time_sleep = 5
296		if DEBUG==True:
297		print("Wait "+str(time_sleep)+" sec for first search . . .")
298		else:
299		time_sleep = randint(min_time, max_time)
300		if DEBUG==True:
301		print("Wait "+str(time_sleep)+" sec for next search . . .")
302		if DEBUG==True:
303		print_line(70,"*")
304		return time_sleep
305
306		def print_line(number=30,char="-"):
307		'''
308		This function print line in screen
309		:param number: number of items in each line
310		:param char: each char of line
311		:return: None
312		'''
313		line=""
314		for i in range(number):
315		line=line+char
316		print(line)
317
318
319		def follow(username):
320		'''
321		This function create following and follower list
322		:param username: username
323		:type username:str
324		:return: (list_1,list_2) as tuple
325		'''
326		print("Collecting Follower Information ...")
327		print_line(70, "*")
328		list_1 = follower_list_gen(username)
329		file = open(username + "_follower.log", "w")
330		print(str(len(list_1)) + " Followers --> " + username + "_follower.log")
331		print_line(70, "*")
332		file.write("\n".join(list_1))
333		file.close()
334		print('Collecting Following Informnation ...')
335		print_line(70, "*")
336		list_2 = following_list_gen(username)
337		file = open(username + "_following.log", "w")
338		print(str(len(list_2)) + " Following --> " + username + "_following.log")
339		print_line(70, "*")
340		file.write("\n".join(list_2))
341		file.close()
342		print('Collecting Stars Informnation ...')
343		print_line(70, "*")
344		stars=star_list(username)
345		file = open(username + "_stars.log", "w")
346		print(str(len(stars)) + " Stars --> " + username + "_stars.log")
347		print_line(70, "*")
348		file.write("\n".join(stars))
349		file.close()
350
351		print('Collecting Repos Informnation ...')
352		print_line(70, "*")
353		repos = repo_list(username)
354		file = open(username + "_repos.log", "w")
355		print(str(len(repos)) + " Repos --> " + username + "_repos.log")
356		print_line(70, "*")
357		file.write("\n".join(repos))
358		file.close()
359		return (list_1,list_2)
360
361		def dif(list_1,list_2):
362		'''
363		This function generate dif files
364		:param list_1:follower list
365		:param list_2: following list
366		:type list_1:list
367		:type list_2:list
368		:return: None
369		'''
370		file = open(username + "_dif1.log", "w")
371		dif_list = list(set(list_2) - set(list_1))
372		print(str(len(dif_list)) + " Following - Not Follower --> " + username + "_dif1.log")
373		print_line(70, "*")
374		file.write("\n".join(dif_list))
375		file.close()
376		file = open(username + "_dif2.log", "w")
377		dif_list = list(set(list_1) - set(list_2))
378		print(str(len(dif_list)) + " Follower - Not Following --> " + username + "_dif2.log")
379		print_line(70, "*")
380		file.write("\n".join(dif_list))
381		file.close()
382		if __name__=="__main__":
383		time_1=time.perf_counter()
384		username=input("Please Enter Your Github Username : ")
385		(list_1,list_2)=follow(username)
386		dif(list_1,list_2)
387		time_2=time.perf_counter()
388		dif_time=str(time_2-time_1)
389		print("Data Generated In "+time_convert(dif_time)+" sec")
390		print("Log Files Are Ready --> " + os.getcwd())
391		gc.collect()
392
393
394
395
396
397
398
399
400
401
402
403
404

sepandhaghighi / gitfollow

Push — master ( a89782...a0e972 )

repo_list() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like