star_list() - Code Metrics - Inspection of "feat : star extractor added" - sepandhaghighi/gitfollow - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( f8bcb7...668301 )

by Sepand

created 2017-05-19 22:35 UTC

star_list() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	3
c	0
b	0
f	0
dl	0
loc	18
rs	9.4285

import requests
import sys
import socket
import os
import datetime
from functools import reduce
import time
from random import randint
DEBUG=False
import gc


def zero_insert(input_string):
    '''
    This function get a string as input if input is one digit add a zero
    :param input_string: input digit az string
    :type input_string:str
    :return: modified output as str
    '''
    if len(input_string)==1:
        return "0"+input_string
    return input_string

def time_convert(input_string):
    '''
    This function convert input_string from uptime from sec to DD,HH,MM,SS Format
    :param input_string: input time string  in sec
    :type input_string:str
    :return: converted time as string
    '''
    input_sec=float(input_string)
    input_minute=input_sec//60
    input_sec=int(input_sec-input_minute*60)
    input_hour=input_minute//60
    input_minute=int(input_minute-input_hour*60)
    input_day=int(input_hour//24)
    input_hour=int(input_hour-input_day*24)
    return zero_insert(str(input_day))+" days, "+zero_insert(str(input_hour))+" hour, "+zero_insert(str(input_minute))+" minutes, "+zero_insert(str(input_sec))+" seconds"

def url_maker_following(Name,page_number):
    '''
    This function return github following page url
    :param Name: Username
    :param page_number: page nubmer of following page
    :type Name:str
    :type Page:int
    :return: github following url as string
    '''
    return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=following"
def url_maker_follower(Name,page_number):
    '''
    This function return github follower page url
    :param Name: username
    :param page_number: page number of follower page
    :type Name:str
    :type page_number:int
    :return: github follower url as string
    '''
    return "https://github.com/" + Name + "?page=" + str(page_number) + "&tab=followers"
def url_maker_star(Name,page_number):
    return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=stars"
def star_extract(input_string):

    '''
    This function extract stared repo from raw_html
    :param input_string: raw input html
    :param follower_name: follower_name
    :type input_string:str
    :type follower_name:str
    :return: user_list as list
    '''
    user_list=[]
    index=0
    while(index!=-1):
        index=input_string.find('<a class="muted-link mr-3',index+33,len(input_string))
        length=input_string[index+33:].find('stargazers">\n')
        star_repo=input_string[index+34:index+33+length]
        if star_repo.find("<svg")==-1 and len(star_repo)!=0:
            user_list.append(star_repo)
    return user_list
def user_list_gen(input_string,follower_name):

    '''
    This function extract usernames from raw_html
    :param input_string: raw input html
    :param follower_name: follower_name
    :type input_string:str
    :type follower_name:str
    :return: user_list as list
    '''
    user_list=[]
    index=0
    while(index!=-1):
        index=input_string.find('alt="@',index+6,len(input_string))
        length=input_string[index+6:].find('"')
        user_name=input_string[index+6:index+6+length]
        if user_name!=follower_name:
            if user_name!=follower_name:
                user_list.append(user_name)
    return user_list[:-1]


def get_html(url):
    '''
    This function extract raw_html file
    :param url: url
    :type url:str
    :return: html data
    '''
    time.sleep(create_random_sleep())
    if internet()==True:
        new_session=requests.session()
        new_session.cookies.clear()
        raw_html=new_session.get(url)
        new_session.close()
        raw_data=raw_html.text
        if "Not Found" in raw_data:
            print("Invalid Github User")
            sys.exit()
        return raw_data
    else:
        print("Error In Internet")
        sys.exit()


def end_check(input_string):
    '''
    This function check end page
    :param input_string: raw html
    :type input_string:str
    :return: True or False
    '''
    if input_string.find("reached the end")!=-1:
        return True
    else:
        return False
def follower_list_gen(follower_name):
    '''
    This function generate follower_list
    :param follower_name: username
    :type follower_name:str
    :return: username follower list
    '''
    follower_list = []
    page_number=0
    while (True):
        page_number += 1
        follower_url = url_maker_follower(follower_name, page_number)
        follower_html = get_html(follower_url)
        if end_check(follower_html) == True:
            break
        temp_list = user_list_gen(follower_html,follower_name)
        follower_list.extend(temp_list)
    return follower_list
def star_list(username):
    '''
    This function return stared_repo list
    :param username: username
    :type username:str
    :return: stared repo as list
    '''
    star_list=[]
    page_number=0
    while (True):
        page_number += 1
        star_url = url_maker_star(username, page_number)
        star_html = get_html(star_url)
        temp_list = star_extract(star_html)
        if len(temp_list)==0:
            break
        star_list.extend(temp_list)
    return star_list

def following_list_gen(follower_name):
    '''
    This function generate following list
    :param follower_name: username
    :type follower_name:str
    :return: username following list
    '''
    following_list = []
    page_number=0
    while (True):
        page_number+=1
        following_url = url_maker_following(follower_name, page_number)
        following_html = get_html(following_url)
        if end_check(following_html) == True:
            break
        temp_list = user_list_gen(following_html,follower_name)
        following_list.extend(temp_list)
    return following_list

def error_log(msg):
    """
    Create the errorlog of the app
    :param msg: error message
    :type msg:str
    """
    if "log" not in os.listdir():
        os.mkdir("log")
    file = open(reduce(os.path.join, [os.getcwd(), "log", "error_log.txt"]), "a")
    file.write(str(datetime.datetime.now()) + " --> " + str(msg) + "\n")
    file.close()

def internet(host="8.8.8.8", port=53, timeout=3):
    """
    Check Internet Connections.
    :param  host: the host that check connection to
    :param  port: port that check connection with
    :param  timeout: times that check the connnection
    :type host:str
    :type port:int
    :type timeout:int
    :return bool: True if Connection is Stable
    >>> internet() # if there is stable internet connection
    True
    >>> internet() # if there is no stable internet connection
    False
    """
    try:
        socket.setdefaulttimeout(timeout)
        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
        return True
    except Exception as ex:
        error_log(str(ex))
        return False

def create_random_sleep(index=1,min_time=1,max_time=3):
    '''
    This function generate sleep time with random processes
    :param index: index to determine first page  and messages(index = 0 is for first page)
    :param min_time: minimum time of sleep
    :param max_time: maximum time of sleep
    :type index:int
    :type min_time:int
    :type max_time:int
    :return: time of sleep as integer (a number between max and min)
    '''
    if index==0:
        time_sleep = 5
        if DEBUG==True:
            print("Wait "+str(time_sleep)+" sec for first search . . .")
    else:
        time_sleep = randint(min_time, max_time)
        if DEBUG==True:
            print("Wait "+str(time_sleep)+" sec for next search . . .")
    if DEBUG==True:
        print_line(70,"*")
    return time_sleep

def print_line(number=30,char="-"):
    '''
    This function print line in screen
    :param number: number of items in each line
    :param char: each char of line
    :return: None
    '''
    line=""
    for i in range(number):
        line=line+char
    print(line)


def follow(username):
    '''
    This function create following and follower list
    :param username: username
    :type username:str
    :return: (list_1,list_2) as tuple
    '''
    print("Collecting Follower Information ...")
    print_line(70, "*")
    list_1 = follower_list_gen(username)
    file = open(username + "_follower.log", "w")
    print(str(len(list_1)) + " Followers --> " + username + "_follower.log")
    print_line(70, "*")
    file.write("\n".join(list_1))
    file.close()
    print('Collecting Following Informnation ...')
    print_line(70, "*")
    list_2 = following_list_gen(username)
    file = open(username + "_following.log", "w")
    print(str(len(list_2)) + " Following --> " + username + "_following.log")
    print_line(70, "*")
    file.write("\n".join(list_2))
    file.close()
    stars=star_list(username)
    file = open(username + "_stars.log", "w")
    print(str(len(stars)) + " Stars --> " + username + "_stars.log")
    print_line(70, "*")
    file.write("\n".join(stars))
    file.close()
    return (list_1,list_2)

def dif(list_1,list_2):
    '''
    This function generate dif files
    :param list_1:follower list
    :param list_2: following list
    :type list_1:list
    :type list_2:list
    :return: None
    '''
    file = open(username + "_dif1.log", "w")
    dif_list = list(set(list_2) - set(list_1))
    print(str(len(dif_list)) + " Following - Not Follower --> " + username + "_dif1.log")
    print_line(70, "*")
    file.write("\n".join(dif_list))
    file.close()
    file = open(username + "_dif2.log", "w")
    dif_list = list(set(list_1) - set(list_2))
    print(str(len(dif_list)) + " Follower - Not Following --> " + username + "_dif2.log")
    print_line(70, "*")
    file.write("\n".join(dif_list))
    file.close()
if __name__=="__main__":
    time_1=time.perf_counter()
    username=input("Please Enter Your Github Username : ")
    (list_1,list_2)=follow(username)
    dif(list_1,list_2)
    time_2=time.perf_counter()
    dif_time=str(time_2-time_1)
    print("Data Generated In "+time_convert(dif_time)+" sec")
    print("Log Files Are Ready --> " + os.getcwd())
    gc.collect()














1		import requests
2		import sys
3		import socket
4		import os
5		import datetime
6		from functools import reduce
7		import time
8		from random import randint
9		DEBUG=False
10		import gc
11
12
13		def zero_insert(input_string):
14		'''
15		This function get a string as input if input is one digit add a zero
16		:param input_string: input digit az string
17		:type input_string:str
18		:return: modified output as str
19		'''
20		if len(input_string)==1:
21		return "0"+input_string
22		return input_string
23
24		def time_convert(input_string):
25		'''
26		This function convert input_string from uptime from sec to DD,HH,MM,SS Format
27		:param input_string: input time string in sec
28		:type input_string:str
29		:return: converted time as string
30		'''
31		input_sec=float(input_string)
32		input_minute=input_sec//60
33		input_sec=int(input_sec-input_minute*60)
34		input_hour=input_minute//60
35		input_minute=int(input_minute-input_hour*60)
36		input_day=int(input_hour//24)
37		input_hour=int(input_hour-input_day*24)
38		return zero_insert(str(input_day))+" days, "+zero_insert(str(input_hour))+" hour, "+zero_insert(str(input_minute))+" minutes, "+zero_insert(str(input_sec))+" seconds"
39
40		def url_maker_following(Name,page_number):
41		'''
42		This function return github following page url
43		:param Name: Username
44		:param page_number: page nubmer of following page
45		:type Name:str
46		:type Page:int
47		:return: github following url as string
48		'''
49		return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=following"
50		def url_maker_follower(Name,page_number):
51		'''
52		This function return github follower page url
53		:param Name: username
54		:param page_number: page number of follower page
55		:type Name:str
56		:type page_number:int
57		:return: github follower url as string
58		'''
59		return "https://github.com/" + Name + "?page=" + str(page_number) + "&tab=followers"
60		def url_maker_star(Name,page_number):
61		return "https://github.com/"+Name+"?page="+str(page_number)+"&tab=stars"
62	View Code Duplication	def star_extract(input_string):
		0 ignored issues – show Duplication introduced 2017-05-19 22:36 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
63		'''
64		This function extract stared repo from raw_html
65		:param input_string: raw input html
66		:param follower_name: follower_name
67		:type input_string:str
68		:type follower_name:str
69		:return: user_list as list
70		'''
71		user_list=[]
72		index=0
73		while(index!=-1):
74		index=input_string.find('<a class="muted-link mr-3',index+33,len(input_string))
75		length=input_string[index+33:].find('stargazers">\n')
76		star_repo=input_string[index+34:index+33+length]
77		if star_repo.find("<svg")==-1 and len(star_repo)!=0:
78		user_list.append(star_repo)
79		return user_list
80	View Code Duplication	def user_list_gen(input_string,follower_name):
		0 ignored issues – show Duplication introduced 2017-05-19 22:36 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
81		'''
82		This function extract usernames from raw_html
83		:param input_string: raw input html
84		:param follower_name: follower_name
85		:type input_string:str
86		:type follower_name:str
87		:return: user_list as list
88		'''
89		user_list=[]
90		index=0
91		while(index!=-1):
92		index=input_string.find('alt="@',index+6,len(input_string))
93		length=input_string[index+6:].find('"')
94		user_name=input_string[index+6:index+6+length]
95		if user_name!=follower_name:
96		if user_name!=follower_name:
97		user_list.append(user_name)
98		return user_list[:-1]
99
100
101		def get_html(url):
102		'''
103		This function extract raw_html file
104		:param url: url
105		:type url:str
106		:return: html data
107		'''
108		time.sleep(create_random_sleep())
109		if internet()==True:
110		new_session=requests.session()
111		new_session.cookies.clear()
112		raw_html=new_session.get(url)
113		new_session.close()
114		raw_data=raw_html.text
115		if "Not Found" in raw_data:
116		print("Invalid Github User")
117		sys.exit()
118		return raw_data
119		else:
120		print("Error In Internet")
121		sys.exit()
122
123
124		def end_check(input_string):
125		'''
126		This function check end page
127		:param input_string: raw html
128		:type input_string:str
129		:return: True or False
130		'''
131		if input_string.find("reached the end")!=-1:
132		return True
133		else:
134		return False
135		def follower_list_gen(follower_name):
136		'''
137		This function generate follower_list
138		:param follower_name: username
139		:type follower_name:str
140		:return: username follower list
141		'''
142		follower_list = []
143		page_number=0
144		while (True):
145		page_number += 1
146		follower_url = url_maker_follower(follower_name, page_number)
147		follower_html = get_html(follower_url)
148		if end_check(follower_html) == True:
149		break
150		temp_list = user_list_gen(follower_html,follower_name)
151		follower_list.extend(temp_list)
152		return follower_list
153		def star_list(username):
154		'''
155		This function return stared_repo list
156		:param username: username
157		:type username:str
158		:return: stared repo as list
159		'''
160		star_list=[]
161		page_number=0
162		while (True):
163		page_number += 1
164		star_url = url_maker_star(username, page_number)
165		star_html = get_html(star_url)
166		temp_list = star_extract(star_html)
167		if len(temp_list)==0:
168		break
169		star_list.extend(temp_list)
170		return star_list
171
172		def following_list_gen(follower_name):
173		'''
174		This function generate following list
175		:param follower_name: username
176		:type follower_name:str
177		:return: username following list
178		'''
179		following_list = []
180		page_number=0
181		while (True):
182		page_number+=1
183		following_url = url_maker_following(follower_name, page_number)
184		following_html = get_html(following_url)
185		if end_check(following_html) == True:
186		break
187		temp_list = user_list_gen(following_html,follower_name)
188		following_list.extend(temp_list)
189		return following_list
190
191		def error_log(msg):
192		"""
193		Create the errorlog of the app
194		:param msg: error message
195		:type msg:str
196		"""
197		if "log" not in os.listdir():
198		os.mkdir("log")
199		file = open(reduce(os.path.join, [os.getcwd(), "log", "error_log.txt"]), "a")
200		file.write(str(datetime.datetime.now()) + " --> " + str(msg) + "\n")
201		file.close()
202
203		def internet(host="8.8.8.8", port=53, timeout=3):
204		"""
205		Check Internet Connections.
206		:param host: the host that check connection to
207		:param port: port that check connection with
208		:param timeout: times that check the connnection
209		:type host:str
210		:type port:int
211		:type timeout:int
212		:return bool: True if Connection is Stable
213		>>> internet() # if there is stable internet connection
214		True
215		>>> internet() # if there is no stable internet connection
216		False
217		"""
218		try:
219		socket.setdefaulttimeout(timeout)
220		socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
221		return True
222		except Exception as ex:
223		error_log(str(ex))
224		return False
225
226		def create_random_sleep(index=1,min_time=1,max_time=3):
227		'''
228		This function generate sleep time with random processes
229		:param index: index to determine first page and messages(index = 0 is for first page)
230		:param min_time: minimum time of sleep
231		:param max_time: maximum time of sleep
232		:type index:int
233		:type min_time:int
234		:type max_time:int
235		:return: time of sleep as integer (a number between max and min)
236		'''
237		if index==0:
238		time_sleep = 5
239		if DEBUG==True:
240		print("Wait "+str(time_sleep)+" sec for first search . . .")
241		else:
242		time_sleep = randint(min_time, max_time)
243		if DEBUG==True:
244		print("Wait "+str(time_sleep)+" sec for next search . . .")
245		if DEBUG==True:
246		print_line(70,"*")
247		return time_sleep
248
249		def print_line(number=30,char="-"):
250		'''
251		This function print line in screen
252		:param number: number of items in each line
253		:param char: each char of line
254		:return: None
255		'''
256		line=""
257		for i in range(number):
258		line=line+char
259		print(line)
260
261
262		def follow(username):
263		'''
264		This function create following and follower list
265		:param username: username
266		:type username:str
267		:return: (list_1,list_2) as tuple
268		'''
269		print("Collecting Follower Information ...")
270		print_line(70, "*")
271		list_1 = follower_list_gen(username)
272		file = open(username + "_follower.log", "w")
273		print(str(len(list_1)) + " Followers --> " + username + "_follower.log")
274		print_line(70, "*")
275		file.write("\n".join(list_1))
276		file.close()
277		print('Collecting Following Informnation ...')
278		print_line(70, "*")
279		list_2 = following_list_gen(username)
280		file = open(username + "_following.log", "w")
281		print(str(len(list_2)) + " Following --> " + username + "_following.log")
282		print_line(70, "*")
283		file.write("\n".join(list_2))
284		file.close()
285		stars=star_list(username)
286		file = open(username + "_stars.log", "w")
287		print(str(len(stars)) + " Stars --> " + username + "_stars.log")
288		print_line(70, "*")
289		file.write("\n".join(stars))
290		file.close()
291		return (list_1,list_2)
292
293		def dif(list_1,list_2):
294		'''
295		This function generate dif files
296		:param list_1:follower list
297		:param list_2: following list
298		:type list_1:list
299		:type list_2:list
300		:return: None
301		'''
302		file = open(username + "_dif1.log", "w")
303		dif_list = list(set(list_2) - set(list_1))
304		print(str(len(dif_list)) + " Following - Not Follower --> " + username + "_dif1.log")
305		print_line(70, "*")
306		file.write("\n".join(dif_list))
307		file.close()
308		file = open(username + "_dif2.log", "w")
309		dif_list = list(set(list_1) - set(list_2))
310		print(str(len(dif_list)) + " Follower - Not Following --> " + username + "_dif2.log")
311		print_line(70, "*")
312		file.write("\n".join(dif_list))
313		file.close()
314		if __name__=="__main__":
315		time_1=time.perf_counter()
316		username=input("Please Enter Your Github Username : ")
317		(list_1,list_2)=follow(username)
318		dif(list_1,list_2)
319		time_2=time.perf_counter()
320		dif_time=str(time_2-time_1)
321		print("Data Generated In "+time_convert(dif_time)+" sec")
322		print("Log Files Are Ready --> " + os.getcwd())
323		gc.collect()
324
325
326
327
328
329
330
331
332
333
334
335
336

sepandhaghighi / gitfollow

Push — master ( f8bcb7...668301 )

star_list() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like