first commit

processdata/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

processdata/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ProcessdataConfig(AppConfig):
+    name = 'processdata'

processdata/getdata.py

@@ -0,0 +1,163 @@
+# ©PSMattas, @psmattas
+# Created: 03-15-2020
+import datetime
+import platform
+
+import pandas as pd
+
+# Datasets scraped can be found in the following URL's:
+# ¹ Johns Hopkins: https://github.com/CSSEGISandData/COVID-19 
+# ² Our World In Data: https://github.com/owid/covid-19-data/tree/master/public/data
+# ³ New York Times: https://github.com/nytimes/covid-19-data
+
+# Different styles in zero-padding in date depend on operating systems
+if platform.system() == 'Linux':
+    STRFTIME_DATA_FRAME_FORMAT = '%-m/%-d/%y'
+elif platform.system() == 'Windows':
+    STRFTIME_DATA_FRAME_FORMAT = '%#m/%#d/%y'
+else:
+    STRFTIME_DATA_FRAME_FORMAT = '%-m/%-d/%y'
+
+
+def daily_report(date_string=None):
+    # Reports aggegrade data, dating as far back to 01-22-2020
+    # If passing arg, must use above date formatting '01-22-2020'
+    report_directory = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/'
+    
+    if date_string is None: 
+        yesterday = datetime.date.today() - datetime.timedelta(days=2)
+        file_date = yesterday.strftime('%m-%d-%Y')
+    else: 
+        file_date = date_string 
+    
+    df = pd.read_csv(report_directory + file_date + '.csv', dtype={"FIPS": str})
+    return df
+
+
+def daily_confirmed():
+    # returns the daily reported cases for respective date, 
+    # segmented globally and by country
+    df = pd.read_csv('https://raw.githubusercontent.com/owid/covid-19-data/master/public/data/jhu/new_cases.csv')
+    return df
+
+
+def daily_deaths():
+    # returns the daily reported deaths for respective date
+    df = pd.read_csv('https://raw.githubusercontent.com/owid/covid-19-data/master/public/data/jhu/new_deaths.csv')
+    return df
+
+
+def confirmed_report():
+    # Returns time series version of total cases confirmed globally
+    df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv')
+    return df
+
+
+def deaths_report():
+    # Returns time series version of total deaths globally
+    df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
+    return df
+
+
+def recovered_report():
+    # Return time series version of total recoveries globally
+    df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv')
+    return df 
+
+
+def realtime_growth(date_string=None, weekly=False, monthly=False):
+    """[summary]: consolidates all reports, to create time series of statistics.
+    Columns excluded with list comp. are: ['Province/State','Country/Region','Lat','Long'].
+
+    Args:
+        date_string: must use following date formatting '4/12/20'.
+        weekly: bool, returns df for last 8 weks
+        monthly: bool, returns df for last 3 months
+    Returns:
+        [growth_df] -- [growth in series]
+    """ 
+    df1 = confirmed_report()[confirmed_report().columns[4:]].sum()
+    df2 = deaths_report()[deaths_report().columns[4:]].sum()
+    df3 = recovered_report()[recovered_report().columns[4:]].sum()
+    
+    growth_df = pd.DataFrame([])
+    growth_df['Confirmed'], growth_df['Deaths'], growth_df['Recovered'] = df1, df2, df3
+    growth_df.index = growth_df.index.rename('Date')
+    
+    yesterday = pd.Timestamp('now').date() - pd.Timedelta(days=1)
+    
+    if date_string is not None: 
+        return growth_df.loc[growth_df.index == date_string]
+    
+    if weekly is True: 
+        weekly_df = pd.DataFrame([])
+        intervals = pd.date_range(end=yesterday, periods=8, freq='7D').strftime(STRFTIME_DATA_FRAME_FORMAT).tolist()
+        for day in intervals:
+            weekly_df = weekly_df.append(growth_df.loc[growth_df.index==day])
+        return weekly_df
+    
+    elif monthly is True:
+        monthly_df = pd.DataFrame([])
+        intervals = pd.date_range(end=yesterday, periods=3, freq='1M').strftime(STRFTIME_DATA_FRAME_FORMAT).tolist()
+        for day in intervals:
+            monthly_df = monthly_df.append(growth_df.loc[growth_df.index==day])
+        return monthly_df
+    
+    return growth_df
+
+
+def percentage_trends():
+    """[summary]: Returns percentage of change, in comparison to week prior.
+    
+    Returns:
+        [pd.series] -- [percentage objects]
+    """    
+    current = realtime_growth(weekly=True).iloc[-1]
+    last_week = realtime_growth(weekly=True).iloc[-2]
+    trends = round(number=((current - last_week)/last_week)*100, ndigits=1)
+    
+    rate_change = round(((current.Deaths/current.Confirmed)*100)-((last_week.Deaths / last_week.Confirmed)*100), ndigits=2)
+    trends = trends.append(pd.Series(data=rate_change, index=['Death_rate']))
+    
+    return trends
+
+
+def global_cases():
+    """[summary]: Creates a table on total statistics of all countries,
+    sorted by confirmations.
+
+    Returns:
+        [pd.DataFrame]
+    """
+    df = daily_report()[['Country_Region', 'Confirmed', 'Recovered', 'Deaths', 'Active']]
+    df.rename(columns={'Country_Region':'Country'}, inplace=True) 
+    df = df.groupby('Country', as_index=False).sum()  # Dataframe mapper, combines rows where country value is the same
+    df.sort_values(by=['Confirmed'], ascending=False, inplace=True)
+    
+    for index, row in df.iterrows():
+        countryCases = int(row['Confirmed'])
+        countryDeaths = int(row['Deaths'])
+        if(countryCases == 0):
+            deathRateFormatted = format(0, '.2f')
+            df.loc[index, 'Death Rate'] = deathRateFormatted
+        else:
+            deathRate = float(countryDeaths / countryCases)*100
+            deathRateFormatted = format(deathRate, '.2f')
+            df.loc[index, 'Death Rate'] = deathRateFormatted
+    return df
+
+def usa_counties():
+    """[summary]: Returns live cases of USA at county-level
+    
+    source:
+        ³ nytimes
+    Returns:
+        [pd.DataFrame]
+    """
+    populations = pd.read_csv('https://raw.githubusercontent.com/balsama/us_counties_data/master/data/counties.csv')[['FIPS Code', 'Population']]
+    populations.rename(columns={'FIPS Code': 'fips'}, inplace=True)
+    df = pd.read_csv('https://raw.githubusercontent.com/nytimes/covid-19-data/master/live/us-counties.csv', dtype={"fips": str}).iloc[:,:6]
+    df = pd.merge(df, populations, on='fips')
+    df['cases/capita'] = (df.cases / df.Population)*100000 # per 100k residents
+
+    return df

processdata/maps.py

@@ -0,0 +1,41 @@
+# File for creation of plotly maps(figs).
+# You can use the plotly builtin fig.show() method to map locally.
+import json
+from urllib.request import urlopen
+
+import plotly.graph_objs as go
+from plotly.offline import plot
+
+from . import getdata
+
+
+def usa_map():
+    # Map of USA subdivided by FIPS-codes (counties), showing cases per-capita basis
+    # Reference: https://plotly.com/python/reference/#choroplethmapbox
+    with urlopen('https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json') as response:
+        counties = json.load(response)
+
+    df = getdata.usa_counties()
+    df.drop([2311], inplace=True)
+
+    fig = go.Figure(
+        go.Choroplethmapbox(
+            geojson = counties, 
+            locations = df.fips,
+            z = df['cases/capita'],
+            marker_opacity = 0.75,
+            marker_line_width = 0,
+            colorscale = [[0, '#FFFAF4'], [.005, '#FFE4CC'], [.030, '#DC654F'], [.060, '#CA3328'], [.080, '#B80000'], [.100, '#7C100C'], [.150, '#580000'], [.175, '#300000'], [1, '#170707']]
+        )
+    )
+
+    fig.update_layout(
+        mapbox_style = 'carto-positron', 
+        paper_bgcolor='rgba(0,0,0,0)', 
+        mapbox_zoom=2.75, 
+        mapbox_center = {'lat': 37.0902, 'lon': -95.7129}, 
+        margin = dict(t=0, l=0, r=0, b=0)
+    )
+    plot_div = plot(fig, include_plotlyjs=False, output_type='div', config={'displayModeBar': False})
+
+    return plot_div

processdata/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

processdata/urls.py

@@ -0,0 +1,13 @@
+from django.urls import path
+from . import views
+
+urlpatterns = [
+    path('', views.index, name='index'),
+    path('maps.html', views.mapspage, name='maps'),
+    path('report', views.report, name='report'),
+    path('trends', views.trends, name='trends'),
+    path('cases', views.global_cases, name='cases'),
+    path('realtime_growth', views.realtime_growth, name='realtime_growth'),
+    path('daily_growth', views.daily_growth, name='daily_growth'),
+    path('daily_report', views.daily_report, name='daily_report')
+]

processdata/views.py

@@ -0,0 +1,91 @@
+from django.http import HttpResponse
+from django.shortcuts import get_object_or_404, redirect, render
+from django.template import loader
+
+import json
+
+from . import getdata, maps
+
+
+def index(request): 
+    return render(request, template_name='index.html')
+
+
+def report(request):
+    df = getdata.daily_report(date_string=None)
+    df = df[['Confirmed', 'Deaths', 'Recovered']].sum()
+    death_rate = f'{(df.Deaths / df.Confirmed)*100:.02f}%'
+
+    data = {
+        'num_confirmed': int(df.Confirmed),
+        'num_recovered': int(df.Recovered),
+        'num_deaths': int(df.Deaths),
+        'death_rate': death_rate
+    }
+
+    data = json.dumps(data)
+
+    return HttpResponse(data, content_type='application/json')
+
+
+def trends(request):
+    df = getdata.percentage_trends()
+
+    data = {
+        'confirmed_trend': int(round(df.Confirmed)),
+        'deaths_trend': int(round(df.Deaths)),
+        'recovered_trend': int(round(df.Recovered)),
+        'death_rate_trend': float(df.Death_rate)
+    }
+
+    data = json.dumps(data)
+
+    return HttpResponse(data, content_type='application/json')
+
+
+def global_cases(request):
+    df = getdata.global_cases()
+    return HttpResponse(df.to_json(orient='records'), content_type='application/json')
+
+
+def world_map():
+    plot_div = maps.world_map()
+    return {'world_map': plot_div}
+
+
+def realtime_growth(request):
+    import pandas as pd
+    df = getdata.realtime_growth();
+
+    df.index = pd.to_datetime(df.index)
+    df.index = df.index.strftime('%Y-%m-%d')
+
+    return HttpResponse(df.to_json(orient='columns'), content_type='application/json')
+
+
+def daily_growth(request):
+    df_confirmed = getdata.daily_confirmed()[['date', 'World']]
+    df_deaths = getdata.daily_deaths()[['date', 'World']]
+
+    df_confirmed = df_confirmed.set_index('date')
+    df_deaths = df_deaths.set_index('date')
+
+    json_string = '{' + \
+        '"confirmed": ' + df_confirmed.to_json(orient='columns') + ',' + \
+        '"deaths": ' + df_deaths.to_json(orient='columns') + \
+    '}'
+
+    return HttpResponse(json_string, content_type='application/json')
+
+
+def daily_report(request):
+    df = getdata.daily_report()
+
+    df.drop(['FIPS', 'Admin2', 'Province_State', 'Country_Region', 'Last_Update', 'Deaths', 'Recovered', 'Active', 'Incident_Rate', 'Case_Fatality_Ratio'], axis=1, inplace=True)
+
+    return HttpResponse(df.to_json(orient='columns'), content_type='application/json')
+
+
+def mapspage(request):
+    plot_div = maps.usa_map()
+    return render(request, template_name='pages/maps.html', context={'usa_map': plot_div})