Machine Learning. Grid search

How to choose best fit parameters for your ANN or SVM model using scikit learn grid search?

(This topic refers to classification problem).
As a beginner, after creating a couple of simple neural networks either via tensorflow or Matlab I was thinking about one question: How do I decide which network architecture (number of layers, neurons) and parameters (epoch number, batch size, optimization algorithms and initialization types) to choose? And actually, this question was answered via Grid Search scikit python library that automatizes process of the best parameters. In example below I tried cover all typical tuning parameters:

# Since running GridSearch may be quite time/calculation consuming  
 # I used GPU based tensorflow in order to speed-up calculation  
 import tensorflow as tf  
 from keras.backend.tensorflow_backend import set_session  
 config = tf.ConfigProto()  
 # Defining GPU usage limit to 75%  
 config.gpu_options.per_process_gpu_memory_fraction = 0.75  
 # Inserting session configuration   
 set_session(tf.Session(config=config))  
 from keras.constraints import maxnorm  
 import numpy  
 from sklearn.model_selection import GridSearchCV  
 from keras.models import Sequential  
 from keras.layers import Dense  
 from keras.wrappers.scikit_learn import KerasClassifier  
 # Function to create neural network model, required for KerasClassifier  
 # Arguments of the function create_model() are those you want to tune  
 def create_model(activation='relu',optimizer='adam',neurons=1,init_mode='uniform'):  
      # create model  
      model = Sequential()  
      model.add(Dense(neurons, input_dim=8, kernel_initializer=init_mode, activation=activation,kernel_constraint=maxnorm(4)))  
      model.add(Dense(1, kernel_initializer=init_mode, activation='sigmoid'))  
      # Compiling model  
      model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])  
      return model  
 # fix random seed for reproducibility  
 seed = 7  
 numpy.random.seed(seed)  
 # load dataset  
 dataset = numpy.loadtxt("some_sample_data.csv", delimiter=",")  
 # split into input (X) and output (Y) variables  
 X = dataset[:,0:8]  
 Y = dataset[:,8]  
 # create model  
 model = KerasClassifier(build_fn=create_model, epochs=100, batch_size=10, verbose=1)  
 # defining the grid search parameters  
 # Here I'm defining a bunch of paramters in a list form  
 batch_size = [10, 20, 40]  
 epochs = [10, 50]  
 init_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']  
 neurons = [1, 5, 10, 15, 20, 25, 30]  
 optimizer = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']  
 activation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']  
 # parameters are fed in a form of dictionary   
 param_grid = dict(activation=activation,optimizer=optimizer,neurons=neurons,init_mode=init_mode,batch_size=batch_size,epochs=epochs)  
 grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=1)  
 grid_result = grid.fit(X, Y)  
 # summarize results  
 print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))  
 means = grid_result.cv_results_['mean_test_score']  
 stds = grid_result.cv_results_['std_test_score']  
 params = grid_result.cv_results_['params']  
 for mean, stdev, param in zip(means, stds, params):  
   print("%f (%f) with: %r" % (mean, stdev, param))  

Along with that you might want to change the number of ANN layers and run code again.
Considering a bigger picture we might think what about SVM classification, what about if SVM model may perform better than ANN? Therefore, it would be helpful to do a GridSearch for Support Vector Machine model:

# Grid Search
 # Importing the libraries  
 import numpy as np  
 import matplotlib.pyplot as plt  
 import pandas as pd  
 # Importing the dataset  
 dataset = pd.read_csv('sample-data.csv')  
 X = dataset.iloc[:, 0:8].values  
 y = dataset.iloc[:, 8].values  
 # Splitting the dataset into the Training set and Test set  
 from sklearn.model_selection import train_test_split  
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = 0)  
 # Feature Scaling  
 from sklearn.preprocessing import StandardScaler  
 sc = StandardScaler()  
 X_train = sc.fit_transform(X_train)  
 X_test = sc.transform(X_test)  
 # Fitting Kernel SVM to the Training set  
 from sklearn.svm import SVC  
 classifier = SVC(kernel = 'rbf', random_state = 0)  
 classifier.fit(X_train, y_train)  
 # Predicting the Test set results  
 y_pred = classifier.predict(X_test)  
 # Making the Confusion Matrix  
 from sklearn.metrics import confusion_matrix  
 cm = confusion_matrix(y_test, y_pred)  
 # Applying k-Fold Cross Validation  
 from sklearn.model_selection import cross_val_score  
 accuracies = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10)  
 accuracies.mean()  
 accuracies.std()  
 # Applying Grid Search to find the best model and the best parameters  
 from sklearn.model_selection import GridSearchCV  
 # Defining parameters list and ranges  
 # C - cost function const; kernel - kernel or similarity funcition;  
 parameters = [{'C': [1, 3,5,10], 'kernel': ['rbf','linear','poly'], 'gamma': [0.1,0.3,0.7, 0.8, 0.9]}]  
 grid_search = GridSearchCV(estimator = classifier,  
               param_grid = parameters,  
               scoring = 'accuracy',  
               cv = 10,  
               n_jobs = 1) # n_jobs=-1 will allow parallel CPU computing;  
 grid_search = grid_search.fit(X_train, y_train)  
 best_accuracy = grid_search.best_score_  
 best_parameters = grid_search.best_params_  
 print(best_accuracy)  
 print(best_parameters)