Анализ ECS 236th Abstracts (1) – Соскабливание данных с BeautifulSoup4

# Load Table of Contents
soup = bs4.BeautifulSoup(open('./ECS MA2019-02.003/Abstracts/tos.html', encoding='utf-8'),'lxml')

# Function to remove html tags
px = re.compile(r"<[^>]*?>")

#Load Categories
p_len = len(soup('h2'))
categorys = []
for i in range(p_len):
    c = px.sub("", str(soup('h2')[i]))
    categorys.append(c)
print("categoly_num ="+str(p_len))
print(categorys)

#Load Sessions
p_len = len(soup('h3'))
sessions = []
for i in range(p_len):
    c = px.sub("", str(soup('h3')[i]))
    sessions.append(c)
print("session_num ="+str(p_len))
print(sessions)

#Load stopwords
with open('stopwords.txt', 'r', encoding='utf-8') as f:
    stops = f.readlines()
stopwords = []
for s in stops:
    s = s.replace("\n", "")
    stopwords.append(s)

dic_all = []

# make session number and name dictionary
dic_sessions = {}
for s in sessions:
    id_ = s.split('-',1)
    dic_sessions[id_[0]]=id_[1]

#Get all pathes of abstract .html file
path = os.listdir("./ECS MA2019-02.003/Abstracts/Html")

#Scraping html documents
for n,i in enumerate(tqdm(path)):
    soup = bs4.BeautifulSoup(open("./ECS MA2019-02.003/Abstracts/Html/"+i, encoding='utf-8'), "html5lib")

    #Get title, authors, affiliation
    title =px.sub("", soup.title.string)
    author = px.sub("", str(soup('p')[1]))
    author = author.split(', ')
    affiliation = px.sub("", str(soup('p')[2]))
    affiliation = affiliation.replace("a", "", 1)

    #Get presentation number, session
    number = re.search('-[0-9]{4}', str(title))
    number = number.group().replace("-", "")
    session = re.match('.*[0-9]{2}', str(title))
    session = session.group()

    #Remove session name in title
    title = re.sub('\w[0-9]{2}-[0-9]{4}', "", title)
    title = title.replace(u"\xa0",u"")
    title = title.replace(session, '')
    title = title.replace('(Invited)', '')

    session = session.replace(number, '')
    session = session.replace('-', '')
    try:
        session_name = dic_sessions[session]
    except:
        session = session.split(u'\xa0',1)[0]
        session_name = dic_sessions[session]

    session_name = session_name.replace(u'&', '&')

    #Get main text and reshape it
    content = px.sub("", str(soup.find(class_=re.compile('contaner_content'))))
    stoplist = [",", ".", ":", ";", "(", ")", "{", "}", "[", "]", "\'", "\""]
    for s in stoplist:
        content = content.replace(s, "")
    content = content.replace("\t", "")

    #Get word list for Word2Vec Learning
    c_list = content.split(" ")
    c_list = [c.lower() for c in c_list]
    for s in stopwords:
        c_list = [c for c in c_list if i != s]
    c_list = [c for c in c_list if c != ""]

    #Summarize it as dictionary
    dic ={'num':number,'title':title,'author':author,
          'affiliation':affiliation,'session':session, 'session_name':session_name,
          'contents':content, 'mod_contents': c_list, 'vector':n, 'url':i}

    dic_all.append(dic)

{'num': '0001',
 'title': 'The Impact of Coal Mineral Matter (alumina and silica) on Carbon Electrooxidation in the Direct Carbon Fuel Cell',
 'author': ['Simin Moradmanda', 'Jessica A Allena', 'Scott W Donnea'],
 'affiliation': 'University of Newcastle',
 'session': 'A01',
 'session_name': 'Battery and Energy Technology Joint General Session',
 'contents': 'Direct carbon fuel cell DCFC as an electrochemical device...',
 'mod_contents': ['direct','carbon','fuel','cell', ... ,'melting'],
 'vector': 0,
 'url': '1.html'}

charactors = [i[0] for i in categorys]
cat_counts = np.zeros(len(charactors))
authors_counts = {}
words_counts = {}

for c in charactors:
    authors_counts[c] = []
    words_counts[c] = []

for d in dic_all:
    char = d['session'][0]
    cat_counts[charactors.index(char)] += 1
    authors_counts[char].append(len(d['author']))
    words_counts[char].append(len(d['contents'].split(' ')))

fig = plt.figure(figsize=(10,4),dpi=200)
plt.bar(charactors, cat_counts, label=categorys)
plt.xlabel('Categories',fontsize=14)
plt.title('Number of presentation',fontsize=16)
plt.show()

points = tuple(authors_counts[i] for i in charactors)
fig = plt.figure(figsize=(10,4),dpi=200)
plt.boxplot(points)
plt.gca().set_xticklabels(charactors)
plt.xlabel('Categories',fontsize=14)
plt.title('Number of author',fontsize=16)
plt.show()

import collections
affiliation_list = []
for a in dic_all:
    affiliation_list.extend(a['affiliation'])   
a_count = collections.Counter(affiliation_list)
a_sorted = sorted(a_count.items(), key=lambda x:x[1], reverse=True)