Twitter Naver YouTube Crawler, 현대자동차 여론을 조사해보자(3)

import os
import twitter

consumer_key = '키입력'
consumer_secret = '키입력'

access_token = '키입력'
access_token_secret = '키입력'

bearer_token = '키입력'

def get_object():
    try:
        api = twitter.Api(consumer_key=consumer_key,
                                  consumer_secret=consumer_secret,
                                  access_token_key=access_token,
                                  access_token_secret=access_token_secret)
        return api
    except Exception as e:
        print(e)
        return None

def get_TwitterSearch(srcText, display, api):
    search = api.GetSearch(term=srcText, count=display)

    print(len(search))
    return search

def UnfoldData(search):
    data = {}  
    i = 0  
    for tweet in search:
        data['text'] = tweet.text  
        print(i, " : ", data)
        i += 1

    return search

def SaveData(search, Header):
    # 전체 문서를 파일 저장
    wfile = open(os.getcwd() + "/" + Header + "twitter.txt", mode='w')  # 쓰기 모드
    data = {}
    i = 0

    for tweet in search:
        data['text'] = tweet.text
        wfile.write(data['text'] + '\n')
        i += 1

    wfile.close()

def main():
    tweety = get_object()

    srcText = input('Enter your target word: ')
    search = get_TwitterSearch(srcText, 100, tweety)
    SaveData(search, srcText)

import os
import sys
import urllib.request
import datetime
import time
import json

client_id = "키입력"
client_secret = "키입력"

def get_RequestURL(url):
    request = urllib.request.Request(url)
    request.add_header("X-Naver-Client-Id", client_id)
    request.add_header("X-Naver-Client-Secret", client_secret)

    try:
        response = urllib.request.urlopen(request)
        rescode = response.getcode()
        if rescode == 200:
            response_body = response.read()
            return response_body.decode('utf-8')
        else:
            return "Error Code:" + rescode
    except Exception as e:
        print(e)
        return None

def get_NaverSearch(node, srcText, page_start, display):
    baseurl = "https://openapi.naver.com/v1/search/"
    node = "%s.json" % node
    qry = "?query=%s&start=%s&display=%s" % (urllib.parse.quote(srcText), page_start, display)

    url = baseurl + node + qry
    responseDecode = get_RequestURL(url)

    if responseDecode == None:
        return None
    else:
        return json.loads(responseDecode)

def get_PostData(post, jsonObj, docNumber):
    title = post['title']
    description = post['description']
    org_link = post['originallink']
    link = post['link']

    pDate = datetime.datetime.strptime(post['pubDate'], '%a, %d %b %Y %H:%M:%S +0900')
    pDate = pDate.strftime('%Y-%m-%d %H:%M:%S')

    jsonObj.append({'cnt':docNumber, 'title':title, 'description': description,
                    'org_link':org_link, 'link':org_link, 'pDate':pDate})
    return

def main():
    # news blog cafearticle movie shop
    node = 'news'
    srcText = input('Enter your target word: ')
    cnt = 0;
    jsonObj = []

    jsonResponse = get_NaverSearch(node, srcText, 1, 100)
    total = jsonResponse['total']

    while((jsonResponse != None) and (jsonResponse['display'] != 0 )):
        for post in jsonResponse['items']:
            cnt += 1
            get_PostData(post, jsonObj, cnt)

        start = jsonResponse['start'] + jsonResponse['display']
        jsonResponse = get_NaverSearch(node, srcText, start, 2700)

    print('Entire searched : %d docs' %total)

    with open('%s_naver_%s.json' %(srcText, node), 'w', encoding='utf8') as outfile:
        jsonFile = json.dumps(jsonObj, indent = 4, sort_keys = True, ensure_ascii = False)

        outfile.write(jsonFile)

    print("Imported data : %d docs" %(cnt))
    print('%s_naver_%s.json SAVED' %(srcText,node))

# 영상취득용
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup

import os
import pandas as pd
import time

def set_Browser(driverpath):
    delay = 3
    # chromedriver를 다운받고 window 사용자는 exe를 붙여야함
    browser = Chrome(driverpath + "/chromedriver")
    browser.implicitly_wait(delay)
    return browser

def handling_Browser(obj, start_url):
    obj.get(start_url)
    obj.maximize_window()

    obj.find_element_by_xpath('//*[@class="scrollable style-scope tp-yt-paper-tabs"]/tp-yt-paper-tab[2]').click()

    body = obj.find_element_by_tag_name('body')  # 스크롤하기 위해 소스 추출

    num_of_pagedowns = 20
    # num_of_pagedowns번 밑으로 내리는 것
    while num_of_pagedowns:
        body.send_keys(Keys.PAGE_DOWN)
        time.sleep(3)
        num_of_pagedowns -= 1

    html0 = obj.page_source
    html = BeautifulSoup(html0, 'html.parser')
    rtnhtml = html.select(
        'h3 > a'
    )

    return rtnhtml

def get_YoutuberClips(srcText):
    path = os.getcwd()

    browser = set_Browser(path)
    start_url = "https://www.youtube.com/user/" + srcText + "/featured"
    html = handling_Browser(browser, start_url)

    title = []
    url = []

    for idx in html:
        if idx.get('href')[:7] != '/watch?':
            pass
        else:
            title.append(idx.text)
            url.append(idx.get('href'))

    title_list = pd.DataFrame(url, columns=['url'])
    title_list['title'] = title
    title_list.to_csv(srcText + "_YouTubeClips.csv")
    return title_list

def get_Replies(info, srcText):
    if len(info.index) == 0:
        info = pd.read_csv(srcText + '_YouTubeClips.csv')

    info['url'] = info['url'].str[9:]

    cnt = 1
    for video_id in info['url']:
        cmd = "python downloader.py --youtubeid " + video_id + " --output result" + str(cnt) + ".csv"
        cnt += 1
        path = os.getcwd()
        os.system("cd " + path)
        os.system(cmd)

    return True

def get_YutubeSearch(srcText):
    os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
    clip_info = get_YoutuberClips(srcText)
    result = get_Replies(clip_info, srcText)
    return result

def Merge_Data(extension, header):
    path = os.getcwd()
    file_list = os.listdir(path)

    file_list_csv = [file for file in file_list if file.endswith(extension)]
    csvset = open('YouTubeResult.txt', 'w')
    for file in file_list_csv:
        f = open(file, 'r')
        subset = f.read()
        csvset.write(subset)
    csvset.close()

def main():
    keyword = 'HyundaiWorldwide'
    result = get_YutubeSearch(keyword)
    Merge_Data('.csv', keyword)

if __name__ == '__main__':
    main()