pullCore/main.py at master · swordjs/pullCore · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# -*- coding: UTF-8 -*-
import sys
from urllib.parse import urlparse
from spiders import github_issues_spider, github_pulls_spider
# from models.model_issue import Issue
# from models.model_pull import Pull

github_source_url = [
            'https://github.com/webVueBlog/interview-answe'
]

def do():
    if len(sys.argv) == 1:
        help()
        return

    param = sys.argv[1]
    if "all" == param:
        crawl_and_scrape('issues')
        crawl_and_scrape('pulls')

    elif "issues" == param:
        crawl_and_scrape('issues')

    elif "pulls" == param:
        crawl_and_scrape('pulls')

    elif "info" == param:
        get_info()
        # issue = Issue()
        # for url in github_source_url:
        #     issue.issue_info("'%s'" % urlparse(url).path.split('/')[2])
    elif "help" == param:
        help()


def help():
    helpMsg = '''usage: python main.py [option]

Select an option parameter to run:
    all     : Get issues and pull requests from github
    issues  : Get issues from github
    pulls   : Get pull requests from github
    info    : Get data info
    help    : For help
    '''
    print(helpMsg)
    return


def crawl_and_scrape(part):
    print(part)
    if 'issues' == part:
        print("进入啊啊***")
        for source in github_source_url:
            url = source + "/" + part
            print('开始爬取%s list: %s' % (part, url))
            issue_list = github_issues_spider.get_issues(url)
            print(issue_list)
            # print('获取list结束，开始获取%s评论...' % part)
            # issue_detail = github_issues_spider.get_all_issues_detail(issue_list)
            # print('获取%s评论信息完毕' % part)

    elif 'pulls' == part:
        for source in github_source_url:
            url = source + "/" + part
            print('开始爬取%s list: %s' % (part, url))
            pull_list = github_pulls_spider.get_pulls(url)
            print('获取list结束，开始获取%s评论...' % part)
            pull_detail = github_pulls_spider.get_all_pulls_detail(pull_list)
            print('获取%s评论信息完毕' % part)


def get_info():
    issue = Issue()
    pull = Pull()
    issue_info = str()
    pulls_info = str()
    for url in github_source_url:
        issue_info += issue.issue_info("'%s'" % urlparse(url).path.split('/')[2]) + "\n\n"
        pulls_info += pull.pull_info("'%s'" % urlparse(url).path.split('/')[2]) + "\n\n"
    with open("issue_info.txt", "wt") as f:
        f.write(issue_info)
    with open("pulls_info.txt", "wt") as f:
        f.write(pulls_info)
    print("Get info successfully!")

if __name__ == '__main__':
    do()
    pass


    # else:
    #     paramError()
    #     return
    # return


    # issue = Issue()
    # res = issue.count('issue', "status='closed'")

    # issue.save_all(issue_detail)