Neighbordhood Venues Categories Pattern vs Neighborhood Prevailing Social Classes

import pandas as pd

bairros = pd.read_csv('bh_bairros_classes.csv') # it was previously saved
bairros.head()

from geopy import Nominatim
from geopy.exc import GeocoderUnavailable, GeocoderTimedOut, GeocoderServiceError

geolocator = Nominatim(user_agent="ny_explorer")

def geolocator_belohorizonte(neighborhood):    
    try:
        locator = geolocator.geocode(neighborhood +', Belo Horizonte')
    except (GeocoderUnavailable, GeocoderTimedOut, GeocoderServiceError):
        # print('Geocoder unavailable or timed out... will try again!')
        locator = geolocator_belohorizonte(neighborhood)
    return locator

print(bairros.shape)
bairros.head()

import folium
import matplotlib.cm as cm
import matplotlib.colors as colors

classes = {'low':0, 'regular':1, 'high':2, 'luxury':3}


ilat = -19.9227318
ilon = -43.9450948

# create map
map_classes = folium.Map(location=[ilat,  ilon], zoom_start=11)

# set color scheme for the clusters
x = np.arange(4)
ys = [i + x + (i*x)**2 for i in range(4)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, socialclass in zip(bairros['Latitude'], bairros['Longitude'], bairros['Neighborhood'], bairros['Class']):
    label = folium.Popup(str(poi) + ' Class ' + socialclass, parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[classes[socialclass]-1],
        fill=True,
        fill_color=rainbow[classes[socialclass]-1],
        fill_opacity=0.7).add_to(map_classes)
    
map_classes

df = bairros[(bairros['Latitude']<ilat+0.09) & (bairros['Latitude']>ilat-0.09) & (bairros['Longitude']>ilon-0.06) & (bairros['Longitude']<ilon+0.06)]
df = df[df['Neighborhood'] != 'PINDORAMA']

categories = {
    'food': '4d4b7105d754a06374d81259',
    'stores': '4d4b7105d754a06378d81259', 
    'profissional': '4d4b7105d754a06375d81259'
}

# function that extracts the category of the venue
def get_category_type(row):
    try:
        categories_list = row['categories']
    except:
        categories_list = row['venue.categories']
        
    if len(categories_list) == 0:
        return None
    else:
        return categories_list[0]['name']
    

# function that returns nearby venues by accessing foursquare    
def getNearbyVenues(names, latitudes, longitudes, category, radius=500, LIMIT=150):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):              
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}&categoryId={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT,
            category
        )
                    
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['id'], 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighborhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'ID',                              
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)

bh_food_venues = getNearbyVenues(names=df['Neighborhood'],
                                   latitudes=df['Latitude'],
                                   longitudes=df['Longitude'],
                                   categories['food']
                                  )

bh_pro_venues = getNearbyVenues(names=df['Neighborhood'],
                                   latitudes=df['Latitude'],
                                   longitudes=df['Longitude'],
                                   categories['professional']
                                  )

bh_stores_venues = getNearbyVenues(names=df['Neighborhood'],
                                   latitudes=df['Latitude'],
                                   longitudes=df['Longitude'],
                                   categories['stores']
                                  )

bh_all_venues = getNearbyVenues(names=df['Neighborhood'],
                                   latitudes=df['Latitude'],
                                   longitudes=df['Longitude'],
                                   ''
                                  )
bh_food_venues.head()

bh_all_venues = pd.read_csv('bh_food_venues.csv')
bh_all_venues.head()

bh_all_venues['Venue Category'].value_counts()

import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn import metrics
from sklearn.linear_model import LogisticRegression

le = preprocessing.LabelEncoder()
le.fit(["low", "regular", "high", "luxury"])

def prepare(df_v):  # returns a dataset with only features and target
    df_venues = df_v

    # one hot encoding
    onehot = pd.get_dummies(df_venues[['Venue Category']], prefix="", prefix_sep="")

    # add neighborhood column back to dataframe
    onehot.insert(0, 'Neighborhood', df_venues['Neighborhood'].values)  

    # creating dataset of onehot categories means
    df_grouped = onehot.groupby('Neighborhood').mean().reset_index()

    # droping neighborhood with less than 8 venues to avoid outliers
    min_venues_mask = df_venues.groupby('Neighborhood').count()['Venue']>=8    
    df_grouped = df_grouped[min_venues_mask.values]

    df_merged = df

    # merge df with bh_food_grouped to add latitude/longitude for each neighborhood
    df_merged = df_merged.join(df_grouped.set_index('Neighborhood'), on='Neighborhood')

    df_merged.dropna(inplace=True)    
    return df_merged.drop(['Latitude', 'Longitude', 'Neighborhood'], 1)


def build_evaluate(X, y):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4)

    LR = LogisticRegression(C=0.1, solver='liblinear', multi_class='auto').fit(X_train, y_train)
    yhat_lr = LR.predict(X_test)
    yhat_lr_prob = LR.predict_proba(X_test)

    print('        Logistic regression accuracy score', metrics.accuracy_score(y_test, yhat_lr))
    print('        Logistic regression log loss', metrics.log_loss(y_test, yhat_lr_prob))

    Ks = 10
    mean_acc = np.zeros((Ks-1))
    std_acc = np.zeros((Ks-1))
    ConfustionMx = [];
    for n in range(1,Ks):

        #Train Model and Predict  
        neigh = KNeighborsClassifier(n_neighbors = n).fit(X_train,y_train)
        yhat=neigh.predict(X_test)
        mean_acc[n-1] = metrics.accuracy_score(y_test, yhat)


        std_acc[n-1]=np.std(yhat==y_test)/np.sqrt(yhat.shape[0])

    print("        KNN accuracy score", mean_acc.max(), " with k=", mean_acc.argmax()+1)
    
    clf = svm.SVC(kernel='rbf', gamma='auto')
    clf.fit(X_train, y_train) 

    yhat_svm = clf.predict(X_test)
    print("        SVM accuracy score", metrics.accuracy_score(y_test, yhat_svm))      

prepare(bh_all_venues)

bh_food_venues = pd.read_csv('bh_food_venues.csv')
bh_pro_venues = pd.read_csv('bh_pro_venues.csv')

names = ['all', 'food', 'professional']
datasets = [bh_all_venues, bh_food_venues, bh_pro_venues]

for name, df_venues in zip(names, datasets):
    
    print('### {} venues category dataset ###'.format(name))
    
    df_merged = prepare(df_venues)

    y = le.transform(df_merged['Class'].values)
    y0 = y == 0
    y1 = y == 1
    y2 = y == 2
    y3 = y == 3

    X = df_merged.drop('Class', 1)

    print('    ### Actual class ###')
    build_evaluate(X, y)

    print('    ### if y == \'low\' ###')
    build_evaluate(X, y0)

    print('    ### if y == \'regular\' ###')
    build_evaluate(X, y1)

    print('    ### if y == \'high\' ###')
    build_evaluate(X, y2)

    print('    ### if y == \'luxury\' ###')
    build_evaluate(X, y3)