"""Questo è il mio primo notebook jupyter, lanciato da anaconda su linux.
Premere CTRL-INVIO per rielaborare una cella

python 3.x si riconosce perchè print è una funzione con gli argomenti tra parentesi
"""
# vedere https://www.w3schools.com/python/default.asp
#
print("hello world!2")

print('caratteri speciali % \ $ \\n va a capo \\t fa il[\t]tab')
print("apice singolo: ['] virgolette [\"]")
print("%d=interi %g=generic number %f=float %s=stringa")
print("intero:[%d] generico:[%g] float:[%f] fload2dec[%.2f]: string:[%s]" % (1, -22.5, 3.0, -3.14159, "ciao"))

a = 25
key = "k%d" % a
print(key)

i=5
print(i)
print("i=%g" % i)

"""
per maggiori info sugli iterators e su yield vedere
https://stackoverflow.com/questions/231767/what-does-the-yield-keyword-do/231855#231855
"""

import itertools
seeds = ["cani", "gatti", "fiori"]  # lista
object = itertools.permutations(seeds) #restituisce una lista di tuple
ordini = list(object)
print("seeds=",seeds)
print("object=",object)
print("ordini=",ordini)
print()
print("len(seeds)=%g" % len(seeds))
print("len(ordini)=",len(ordini))

# regular expressions
#
import re
s = "13-11g-1966"
p1 = re.compile('\s')      # whitespace matcher (space, tab, cr, lf, etc)
s1 = p1.sub('',s)          # replace whitespace with nothing (remove whitespace)
p2 = re.compile('[^0-9\-]') 
s2 = p2.sub('',s1)         # remove everything except digits and "-"
print("s2: [%s]" % s2)

# regular expressions, strings and date parsing
#
import re #regular expressions see https://docs.python.org/3/howto/regex.html

def dmy2ymd(s):
    # transforms " 21-8-80  " in "1980-08-21"
    p2 = re.compile('[^0-9\-]') 
    s2 = p2.sub('',s)         # remove everything except digits and "-"
    print("s2: [%s]" % s2)
    s2l = re.split('[\-]',s2) # create a list splitting on "-" (put in [] the chars you want to split on escaping with \ if needed)
    d = int(s2l[0]) # as numbers
    m = int(s2l[1])
    y = int(s2l[2])
    print("received: [%s] d=[%g] m=[%g] y=[%g]" %(s,d,m,y))
    if not d in range(1,31):
        print("dmy2ymd: day parsing error: setting day to 1")
        d = 1
    if not m in range(1,12):
        print("dmy2ymd: month parsing error: setting month to 1")
        m = 1
    if y in range(0,99):
        y += 1900
    if not y in range(1900,2050):
        print("dmy2ymd: year parsing error: setting year to 2000")
        y = 2000
    # si potrebbero fare i controlli sul numero dei giorni dei vari mesi e sui bisestili    
    dd = str(d)
    if len(dd) == 1: dd = "0" + dd
    mm = str(m)
    if len(mm) == 1: mm = "0" + mm
    yyyy = str(y)
    out = yyyy + "-" + mm + "-" + dd
    return out

print(dmy2ymd("12-11-1980"))
print(dmy2ymd("  29-13 -1981"))
print(dmy2ymd(" 32-11   -84"))
print(dmy2ymd(" 32-11 g -2014"))

# liste di tuple
# see https://docs.python.org/3/tutorial/datastructures.html?highlight=tuples#tuples-and-sequences
#
import operator # required for sorting using itemgetter

db0 = [ # name, surname, birthdate gg-mm-yyyy, score
    ("Marco","Guardigli","13-11-1966",12),
    ("Roberto","Guardigli","03-01-1969",17),
    ("Zorro","Zeta","01-01-1980",8),
    ("Zenda","Zeta","02-01-1981",12)
    ]
db0.insert(1,("Klaudija","Stefan","14-01-1972",21)) # insert a new tuple in the list before the specified element (zero based)

print(db0)
db1 = [] # empty list
id = 0 # unique id will mark each tuple in the new list db1
for k in db0:
    #      id, name, surname, birthdate gg-mm-yyyy, score
    k1 = ((id , k[0], k[1], k[2], k[3]))
    db1.append((k1)) # adds the new tuple at the end of the new list
    id += 1
print(db1) 

# now will sort the list in reverse surname,name,score order, with in-place-sort, using operator itemgetter
db1.sort(key=operator.itemgetter(2,1,4))
# db1.sort(key = lambda tup:tup[2] , reverse=True) # this fails with identical sort keys
print(db1)

# reverse sorting in a new list, without operator.itemgetter, using lambda sort function with multiple keys
db2 = sorted(db1, key = lambda x: (x[2] , x[1] , x[4]), reverse=True)
print(db2)

# ranges & zip & list cast
nums = [x for x in range(1,6)] # range va da zero o primo valore all'(ultimo -1)
vowels = [v for v in "aeiou"]
mapping = dict(zip(nums,vowels)) # crea un array associativo con coppie chiave-valore prese da nums e vowels
r = range(1,4)

print(nums)
print(vowels)

for k in r: print(k,vowels[k])

print(list(vowels[j] for j in r))

for i in mapping:
    print(i, mapping[i])

# http read
# character frequency analysis
#
import urllib.request
import sys               # per getsizeof

contents = urllib.request.urlopen("https://www.google.com/").read()
print(type(contents))
print(sys.getsizeof(contents))
print()
print(contents[0:100]) # visualizza i primi 100 bytes
print()

# ora costruiamo un diagramma delle frequenze dei vari simboli letti
print("frequenze dei vari simboli:")
freq = {} # uso array associativo (key,item) per calcolare le frequenze dei vari simboli
for i in contents:
    k = "k%s" % i    # "k65" e' la chiave il cui valore è la frequenza di "A" il cui byte è 65 
    if k in freq: freq[k] += 1    
    else: freq[k] = 1
for k in freq:
    print("[%s]: %g" % (k, freq[k]))
#print(freq)
    

# http read

import urllib.request
import sys
import operator
# from operator import itemgetter, attrgetter  #per il sorting

contents = urllib.request.urlopen("https://www.google.com/").read()
print(contents[0:100]) # visualizza i primi 100 bytes
print()

# ora costruiamo un diagramma delle frequenze dei vari simboli letti
print("frequenze dei vari simboli:")
freq = [] # uso un array di tuple in cui il primo elemento è il codice carattere e il secondo è la frequenza
for i in contents:
    k = "k%s" % i    # "k65" e' la chiave il cui valore è la frequenza di "A" il cui byte è 65
    if k in freq: freq[k] += 1
    else: freq[k] = 1
for k in freq:
    print("[%s]: %g" % (k[0], k[1]))

    #print(freq)
# ordiniamo per frequenza decrescente. la funzione sorted() produce una lista ordinata a partire da un iterabile
#sorted_keys = sorted(freq) #ordina per valori crescenti di key
#sorted_keys_rev = sorted(freq, reverse=True) #ordina per valori decrescenti di key
#sorted_keys_byitem = sorted(freq, key=lambda : valore

                            
                            

# passaggio parametri
# python talvolta passa per valore e talvolta per reference, a seconda del tipo e le conseguenze degli operatori
# possono essere poco prevedibili

def nplus1(x):
    print("  nplus1 id(x)", id(x))
    x = x + 1 # this x is always a local copy
    return x
def nplus2(x):
    print("  nplus2 id(x)", id(x))
    x += 1 # this x is always a local copy
    return x

def lplus1(x):
    print("  lplus1 id(x)", id(x))
    x = x + ["extra"] # this alters a local copy of the list object
    return x
def lplus2(x):
    print("  lplus2 id(x)", id(x))
    x += ["extra"]  # the in place operator += alters the passed list object (uses its pointer)
    return x


a = 3
b = 0
print("before nplus1(a):",a,b)
b = nplus1(a)
print("after nplus1(a):",a,b)

a = 3
b = 0
print("before nplus2(a):",a,b)
b = nplus2(a)
print("after nplus2(a):",a,b)

a = ["uno","due"]
b = []
print("before lplus1(a):", a , id(a) , b)
b = lplus1(a)
print("after lplus1(a):", a , id(a) , b)

a = ["uno","due"]
b = []
print("before lplus2(a):", a , id(a) , b)
b = lplus2(a)
print("after lplus2(a):", a , id(a) , b)

# see https://www.python-course.eu/python3_passing_arguments.php

import timeit     #misurazione tempi
# quando si importa una libreria, si può in generale vederne l'help eseguendo help(nomelib)
import itertools
import math
seeds = "abcdefghijk"
"""
con 10 caratteri, senza stampare, ci mette circa 3.2 sec a calcolare 3.6 Milioni di permutazioni
con 11 caratteri, senza stampare, ci mette circa 39 sec a calcolare circa 40 milioni di permutazioni

"""
size = math.factorial(len(seeds))
ordini = [] #array di risultati

time_start = timeit.default_timer()      # prende tempo 1
for s in itertools.permutations(seeds):
    o = ""
    for i in s:
        o = o + i #concatenazione di stringhe
    ordini.append(o)
time_end = timeit.default_timer()        # prende tempo 2
print("Elapsed (secs): %.2f" % (time_end - time_start))     #mostra tempo trascorso
print("seeds=",seeds)
print("size=",size)
#print("ordini=",ordini)
print()
print("len(seeds)=%g" % len(seeds))
print("len(ordini)=",len(ordini))

"""
stringa.format(args) produce la sostituzione dei parametri tra graffe con i suoi argomenti
Per rappresentare una parentesi graffa se ne mettono due di fila
"""
i=5
s1 = "the value of i is {0}\nthe second and third args follow: {1},{{{2}}}"
s2 = s1.format(i,2,"tre")
print(s2)

i=6
if i == 5:
    print(" i is 5")
    
if i==6:
        print("i is 6")

def mult(x,y):
    return x*y

print(mult(2,3))

from random import random

def tellme(s):
    answer = ""
    # random() gives back a floating number between 0 and 1
    if random() > 0.3:
        answer = " is great"
    else:
        answer = " is not great"
    return s + answer


mylist = ("Marco", "Klaudija", "Loti")

for me in (mylist):
    print(tellme(me))
    

# experiment with randomness
from random import random

linelength = 60

for j in range(0,10):
    i = 0
    myline = ""
    while i < linelength:
        if random() > 0.5:
            sym = "@"
        else:
            sym = "-"
        myline += sym    
        i += 1
    print(j,myline)

# funzione fattoriale ricorsiva
def fact(x):
    if x == 1: return 1
    else: return x * fact(x-1)
    
for a in range(1,11): #range parte da 0 se non si specifica il primo argomento, e si ferma al valore indicato -1
    print("fact(%d)=%d" % (a,fact(a)))

# funzione fibonacci ricorsiva
def fibo(x):
    if x == 0: return 0
    if x == 1: return 1
    else: return (fibo(x-2) + fibo(x-1))

for a in range(1,100):
    print("fibo(%g)=%g" % (a, fibo(a)))
    

"""Operatori"""
print(2+3,2-3,2*3,2/3)
print(5//2,8%3,5**3)
x = 2
print("x=%g" % x)
x += 6
print("x=%g" % x)
x %= 3
print("x=%g" % x)
x **= 3
print("x=%g" % x)
x //= 2
print("x=%g" % x)
x -= 3
print("x=%g" % x)
print("-=-")
x = 5
print("x=%g" % x)
x -=- 1 # sottrae da x -1 quindi incrementa x di 1 (un po' strano ma possibile e sintatticamente corretto)
print("x=%g" % x)
x -=- x # statement "simmetrico" per raddoppiare x
print("x=%g" % x)
print()
print("logical")
print(not True)
print(True or False)
print( 1 != 2 )
print( 1 and 2 ) 
print( 1 | 2 ) # bitwise or
print( 1 & 2 ) # bitwise and
print( ~2 )    # (cambio segno e aggiungo - 1 (bitwise))
print( 1 << 12 ) # bitwise shift left
print( 2 ^ 255 ) # bitwise xor

# 
# vedere https://www.programiz.com/python-programming/operators
#

# input da utente
#
n = input("inserire il nome:")
print("ciao [%s]" % n)

# list input ed emulazione di ciclo do--while (cond)
#
myl = []
print("inserisci un elemento per volta, invio per terminare")
while True:
    e = input(":")
    if e == "" : break 
    if e == "007": continue # this secret code skips append
    myl.append(e)
print("myl=", myl)
    

# list operators
# append e extend
myl = []
for x in ("uno","due","tre"): # tra parentesi tonde è una tupla. le tuple sono immutabili
    myl.append(x)
print(myl)
ext1 = "quattro","cinque" 
myl.append(ext1)
ext2 = ("sei","sette") # una tupla
myl.append(ext2)
ext3 = ["otto","nove"] # una lista
myl.append(ext3)
ext4 = ["dieci", "undici"] 
myl.extend(ext4) # extend attacca una lista alla fine una lista

print(myl)
for e in myl:
    print(e)
    
print("--")

myl.pop(3) # elimina un elemento e shifta i successivi
myl.pop(3) # elimina un elemento
myl.pop(3) # elimina un elemento
for e in myl:
    print(e)

print("--")
myl.remove("tre") # rimuove la prima occorrenza e da errore se l'elemento non c'e'
myl.insert(2,"tre") # inserisce e sposta in avanti i successivi
for x in ("sei","cinque","quattro"):
    myl.insert(3,x)
l=["sette","otto","nove"]
l.reverse()
for x in l:
    myl.insert(6,x)

myl.pop(len(myl)-1) # elimina ultimo elemento
for e in myl:
    print(e)

# for loops with chars
#
for c in "123!\"£$":
    print(c)

for i in range(65,70):
    print(chr(i))

# try statement
# 
# try:
#     operation_that_can_throw_ioerror()
# except IOError:
#     handle_the_exception_somehow()
# else:
#     # we don't want to catch the IOError if it's raised
#     another_operation_that_can_throw_ioerror()
# finally:
#     something_we_always_need_to_do()
#
print("all ok")
try:
    print(1/0)
except TypeError:
    print("  siamo nel pass")
    pass
except:
    print("  uh oh!")
    
    
print("continuiamo")

# function indexing 
#

def oper(f,x,y):
    def plus(x,y):
        return x+y
    def minus(x,y):
        return x-y
    def mul(x,y):
        return x*y
    def div(x,y):
        return x/y
    def otherwise(x,y):
        return 0
    switcher = { "p":plus, "m":minus, "b":mul, "d":div } # array associativo
    func = switcher.get(f,otherwise)
    return func(x,y)

print(oper("p",2,3))
print(oper("m",2,3))
print(oper("b",2,3))
print(oper("d",2,3))
print(oper("x",2,3))

# function names
#
def plus1(x):
    x += 1
    return x

add1 = plus1 # assegno nomi di funzioni
print("add1", type(add1), id(add1))
print("plus1", type(plus1), id(plus1))

print(add1(3))

del plus1

try:
    print("plus1", type(plus1), id(plus1))

print(add1(13))

# uso di decorators (function wrappers)
# see https://www.python-course.eu/python3_decorators.php
# timing execution
#

import time

def time_usage(func):
    def wrapper(*args, **kwargs):   # *args means one or more arguments
        beg_ts = time.time()
        retval = func(*args, **kwargs)
        end_ts = time.time()
        print("  elapsed time: %.4f" % (end_ts - beg_ts))
        return retval
    return wrapper

@time_usage
def test(n):
    a = 0
    for i in range(0, n):
        a = a + 109*i - i^2 + i^3 
    return a
        
print("inizio")
x = 100000
s1 = time.time()    # seconds since epoch (1.1.1970 00:00)
print("  test(%g)=%d" % (x, test(x)))
s2 = time.time()
print("  now: [%s] elapsed: %.4f" % (time.ctime(s2), s2-s1))
print("fine")

# python type
# Text Type: 	str
# Numeric Types: 	int, float, complex
# Sequence Types: 	list, tuple, range
# Mapping Type: 	dict
# Set Types: 	set, frozenset
# Boolean Type: 	bool
# Binary Types: 	bytes, bytearray, memoryview
#   

t_str = "string"
t_int = 42
t_float = 2.71828
t_complex = 2+.3j
t_list = ["uno","due","tre"]
t_tuple = ("primo","secondo")
t_range = range(0,6)  
t_dict = dict(name="Marco", age=53)
t_bool = True
t_bytes = 122
t_bytearray = b"Hello World."
t_memoryview = memoryview(t_bytearray)

if not t_memoryview.readonly:
    t_memoryview[-1] = '!'

def print_type(x):
    print(x , " is a ", type(x))

mytuple = tuple((t_str, t_int, t_float, t_complex, t_list, t_tuple, t_range, t_dict, t_bool, t_bytes, t_bytearray, t_memoryview))

for i in mytuple:
    print_type(i)

if isinstance(t_str, str):
    print("t_str is a str type")
    
print()
print(t_bytearray)

import os
from glob import glob
import matplotlib.pyplot as plt
import random
import cv2
import pandas as pd
import numpy as np
import matplotlib.gridspec as gridspec
import seaborn as sns
import zlib
import itertools
import sklearn
import itertools
import scipy
import skimage
from skimage.transform import resize
import csv
from tqdm import tqdm
from sklearn import model_selection
from sklearn.model_selection import train_test_split, learning_curve,KFold,cross_val_score,StratifiedKFold
from sklearn.utils import class_weight
from sklearn.metrics import confusion_matrix
import keras
from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D, Lambda, MaxPool2D, BatchNormalization
from keras.utils import np_utils
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras import models, layers, optimizers
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score
from sklearn.utils import class_weight
from keras.optimizers import SGD, RMSprop, Adam, Adagrad, Adadelta, RMSprop
from keras.models import Sequential, model_from_json
from keras.layers import Activation,Dense, Dropout, Flatten, Conv2D, MaxPool2D,MaxPooling2D,AveragePooling2D, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
from keras import backend as K
from keras.applications.vgg16 import VGG16
from keras.models import Model
from keras.applications.inception_v3 import InceptionV3
from imblearn.over_sampling import RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler
#from keras.applications.mobilenet import MobileNet
#from sklearn.metrics import roc_auc_score
#from sklearn.metrics import roc_curve
#from sklearn.metrics import auc
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")