/percolation.py (a32ff93ffd1cfd5603eb93aa4bd63b921c93ee66) (10925 bytes) (mode 100644) (type blob)

import graph_tool.all as gt
import numpy as np
import datetime as dt
import visualization

###############################################
### NOTE: We use the largest component, not ###
### the entire graph for the calculation    ###
###############################################

###############################################
### functions used by all percolation modes ###
###############################################
def percolation(percolated_graph,intact_graph):
  return 100.0*(1.0-float(percolated_graph.num_vertices())/float(intact_graph.num_vertices()))

def print_info(flc, glc):
  print 'filtered graph - vertices: '+str(flc.num_vertices())+' / edges: '+str(flc.num_edges())
  print 'percolation: '+str(percolation(flc,glc))+'%'

# the function below was needed in previous versions of CoRiA because the set of set members was nested within another set
#def read_redis_smembers(redis,key):
 # s = redis.smembers(key) #read set
  #return [i.strip() for i in [l.strip('[]').split(',') for l in s][0]] #write list and strip of useless characters


#################################
####### percolation modes #######
#################################

# These percolation modes take as input the mode name and n - a dictionary of
# numbers of nodes to take out (as keys) and corresponding percentages.
# They return a dictionary of percentage keys and percolation values.
# Advanced percolation modes nest this dictionary within a dictionary of groups.
# Therefore, they require a loop over these groups, which can be e.g. metrics or countries.

#################################
#### BASIC PERCOLATION MODES ####
#################################
def failure(self, mode_name, n):
  print 'Calculating percolation due to random failure'
  # initialise
  counter = 0
  results = {}

  # take a random sample from the largest component
  for v in np.random.choice(list(self.glc.vertices()),size=max(n.keys()),replace=False):
    self.exclusion_map[self.g.vertex(v)] = 0
    counter += 1
    if counter in n.keys():
      print counter,'nodes removed'
      # graph without the excluded vertices (i.e. those that have value 0 in the exclusion map)
      f = gt.GraphView(self.g, vfilt = self.exclusion_map)
      # largest component of graph f
      flc = gt.GraphView(self.g, vfilt = gt.label_largest_component(f))
      print_info(flc,self.glc)
      results[n[counter]] = percolation(flc,self.glc)
      # visualize deterioration
      # visualization.draw_deterioration(self,self.sfdp,mode_name+'_'+str(int(n[counter]))+'_pct')

  return results

#####################################################################

def random_walk(self, mode_name, n):
  print 'Calculating percolation due to random walk'
  #first vertex for random walk
  start = self.glc.vertex(np.random.randint(0,self.glc.num_vertices()), use_index=False)

  #do random walk
  alternate_list = list(self.label_map.a)
  np.random.shuffle(alternate_list)
  results = rw(self,start,n,alternate_list,mode_name)

  #return dict(zip(percentages,percolations))
  return results

#####################################################################

#################################
## ADVANCED PERCOLATION MODES ###
#################################
def target_list(self, mode_name, n):
  print 'Calculating percolation due to targeted attack , i.e. taking out top nodes from a target list'
  # instantiate results dictionary and target lists
  results = {}
  nodes_max = {}

  #loop through all metrics
  all_metrics = list(self.base_metrics.keys() + self.advanced_metrics.keys())
  for metric in all_metrics:
    #get nodes with highest value of metric
    nodes_max[metric] = self.redis.zrange(self.metric_prefix+metric+self.normalization_suffix, -max(n.keys()), -1, withscores=False, score_cast_func=float).reverse()
  #loop through all scores
  all_scores = list(self.scores.keys() + self.advanced_scores.keys())
  for score in all_scores:
    #get nodes with highest value of score
    nodes_max[score] = self.redis.zrange(self.score_prefix+score, -max(n.keys()), -1, withscores=False, score_cast_func=float).reverse()

  #loop through all metrics and scores
  for metric in all_metrics+all_scores:
    print 'Taking out top nodes for metric',metric

    # initialise variables and exclusion map
    counter = 0
    self.exclusion_map.a = 1
    results[metric] = {}

    for node in nodes_max[metric]:
      vertex = gt.find_vertex(self.g,self.label_map,node)[0]
      self.exclusion_map[vertex] = 0
      counter += 1
      if counter in n.keys():
        print counter,'nodes removed'
        # graph without the excluded vertices (i.e. those that have value 0 in the exclusion map)
        f = gt.GraphView(self.g, vfilt = self.exclusion_map)
        # largest component of graph f
        flc = gt.GraphView(self.g, vfilt = gt.label_largest_component(f))
        print_info(flc,self.glc)
        results[metric][n[counter]] = percolation(flc,self.glc)
        # visualize deterioration
        # visualization.draw_deterioration(self,self.sfdp,mode_name+'_'+metric+'_'+str(int(n[counter]))+'_pct')

  return results

#####################################################################

def hybrid_mode(self, mode_name, n):
  print 'Calculating percolation due to random walk starting from node with highest value of metric'
  # instantiate results dictionary and alternate lists for random walk
  results = {}
  alternate_lists = {}

  #loop through all metrics
  all_metrics = list(self.base_metrics.keys() + self.advanced_metrics.keys())
  for metric in all_metrics:
    #get nodes with highest value of metric
    temp_list = self.redis.zrange(self.metric_prefix+metric+self.normalization_suffix, 0, -1, withscores=False, score_cast_func=float)
    alternate_lists[metric] = [node for node in reversed(temp_list)]

  #loop through all scores
  all_scores = list(self.scores.keys() + self.advanced_scores.keys())
  for score in all_scores:
    #get nodes with highest value of score
    temp_list = self.redis.zrange(self.score_prefix+score, 0, -1, withscores=False, score_cast_func=float)
    alternate_lists[score] = [node for node in reversed(temp_list)]

  #loop through all metrics and scores
  for metric in all_metrics+all_scores:
    print 'Starting from node with highest value of metric',metric
    #initialise exclusion vertex property map
    self.exclusion_map.a = 1

    #first vertex for random walk
    start = gt.find_vertex(self.g,self.label_map,alternate_lists[metric][0])[0]

    #do random walk
    results[metric] = rw(self,start,n,alternate_lists[metric],mode_name+'_'+metric)

  return results

def russian(self, mode_name, n):
  print 'Calculating percolation due to shutting off the Russian network from the internet'
  # instantiate results dictionary and target lists
  #results = {}
  #nodes_max = {}
  self.exclusion_map.a = 0
  counter = 0
  results = {}
  for v in self.g.vertices():
    if self.g.vp.country_code[v] == 'RU':
      print 'Shutting off node',int(v),'because it\'s Russian!'
      self.exclusion_map[v] = 1
      counter += 1
#      if counter in n.keys():
#        print counter,'nodes removed'
        # graph without the excluded vertices (i.e. those that have value 0 in the exclusion map)
#        f = gt.GraphView(self.g, vfilt = self.exclusion_map)
        # largest component of graph f
#        flc = gt.GraphView(self.g, vfilt = gt.label_largest_component(f))
#        print_info(flc,self.glc)
#        results[n[counter]] = percolation(flc,self.glc)
        # visualize deterioration
        # visualization.draw_deterioration(self,self.sfdp,mode_name+'_'+metric+'_'+str(int(n[counter]))+'_pct')

  f = gt.GraphView(self.g, vfilt = self.exclusion_map)
  flc = gt.GraphView(self.g, vfilt = gt.label_largest_component(f))
  #results[max(n.values())] = percolation(flc,self.g)
  # visualize deterioration
  print 'Creating visualization #1 of the deterioration.'
  visualization.draw_deterioration(self,self.g.vp.sfdp,mode_name+"_SFDP_inverse")
  print 'Creating visualization #2 of the deterioration.'
  visualization.draw_deterioration(self,self.g.vp.Random,mode_name+"_Random_inverse")
  print 'Creating visualization #3 of the deterioration.'
  visualization.draw_deterioration(self,self.g.vp.Radial,mode_name+"_Radial_inverse")
  #return results
#####################################################################
############## Random Walk for the RW deletion modes ################
#####################################################################

# takes as input a start vertex, the number of vertices to take out
# and an alternate list of vertices if the random walk reaches a dead end

def rw(self, vertex, n, alternate_list, mode_name):
  # initialise
  results = {}

  self.exclusion_map[vertex] = 0 #take out start vertex
  # initialise graph filters
  # graph without the excluded vertices (i.e. those that have value 0 in the exclusion map)
  f = gt.GraphView(self.g, vfilt = self.exclusion_map)
  # largest component of graph f
  flc = gt.GraphView(self.g, vfilt = gt.label_largest_component(f))
  if 1 in n.keys():
    print '1 node removed'
    print_info(flc,self.glc)
    results[n[1]] = percolation(flc,self.glc)
    # visualize deterioration
    # visualization.draw_deterioration(self,self.sfdp,mode_name+'_'+str(int(n[1]))+'_pct')

  for i in range(max(n.keys())-1):
    neighbours = list(vertex.all_neighbours())
    flag = 0 #decision flag

    # choose a random neighbour
    if len(neighbours) > 0:
      np.random.shuffle(neighbours)
      for neighbour in neighbours:
        if self.exclusion_map[neighbour] != 0:
          vertex = neighbour
          flag = 1
          break

    # to be executed if no neighbours exist - choose the next node out of an alternative list
    if flag == 0:
      # create a list of already used list members
      used_list = []
      for node in alternate_list:
        vertex = gt.find_vertex(self.g,self.label_map,node)[0]
        used_list.append(node)
        if self.exclusion_map[vertex] != 0:
          break
      if len(used_list) > 0:
        for used_node in used_list:
          # remove used members from alternate list. This reduces calculation time in next iteration
          alternate_list.remove(used_node)

    self.exclusion_map[vertex] = 0 #take out vertex
    f = gt.GraphView(self.g, vfilt = self.exclusion_map) #update graph (filtered)
    if i+2 in n.keys():
      flc = gt.GraphView(self.g, vfilt = gt.label_largest_component(f)) #update largest component
      print i+2,'nodes removed'
      print_info(flc,self.glc)
      results[n[i+2]] = percolation(flc,self.glc)
      # visualize deterioration
      # visualization.draw_deterioration(self,self.sfdp,mode_name+'_'+str(int(n[i+2]))+'_pct')

  return results

##############################
##############################
########## THE END ###########
##############################
##############################



Mode Type Size Ref File
100644 blob 6 0d20b6487c61e7d1bde93acf4a14b7a89083a16d .gitignore
100644 blob 112 cb89d830628675849b4fc506e25faaee9ada3055 README.md
100644 blob 39806 a3ed366a3ca7fa6017d524c37cb15d492ef7a0ab RU_ASN.txt
040000 tree - fe0210446ce6fae4108a4700bb1ca9d427592d96 RedisHelpers
100644 blob 1256 489636a0073e3dfe2bfd04ee893d609d304a8490 advancedscores.py
100644 blob 1175 51d93965eefe7cc74022747e2b9207a5d619dacc advancedscores.pyc
100644 blob 1420 985dee55c7d8134c4860db8bcccced703b2eb0af change_output_file.py
100644 blob 7893 52fc687151813bb18b8d08a6bcd5b4ea220ad2b1 config.py
100644 blob 2752 04b1176c3586c57b214cf3b20c110c4bd0956d25 config.pyc
040000 tree - 085c594868d9df498c937b91dc7a43e02fb9a934 data
100644 blob 4571 88c66d8e136505e5a29e3aeb8e6e7e131424f4e3 file_importer.py
100644 blob 2190 9b19358d5f4c39da2da330a4dba4416ae089ff61 file_importer.pyc
100644 blob 633 df288a704ff3390fbe5f3090d488fe45df79c23a graph_test.py
100644 blob 901 ae318bee51ebd11d4524c90cc13193f7e8bc208f gt_file_importer.py
100644 blob 1366 12f8577e0a31cee71b70c20849d18ebd36174796 gt_file_importer.pyc
100644 blob 2064 728489f88ef880350a83f4ab8b1e0510239ce067 gt_start.py
100644 blob 858 28f726f3a2fe224a374f24b299fdfdc8bef9f569 indexing.py
100644 blob 1571 68587bbbe3043dbed78d9b3b3018e802e93c6931 indexing.pyc
100644 blob 33232205 7ca601d1ca32abf3244359ce3ad85ea6a1b60010 log
100644 blob 15904 733b22f32ecb738568099b60b22c781df40dd067 metric_calculator.py
100644 blob 11204 b7d78f3bc365a4639c4b462f1724b732d8a11122 metric_calculator.pyc
100644 blob 14963 6692689a02b4cca1b60e568579723eedb748beec metrics.py
100644 blob 11616 a56470059e373ca2da9ac235a5457235a3a2fef8 metrics.pyc
100644 blob 61770227 1adc4f608c6a5fb41881af629b708af873dfed5f nohup.out
100644 blob 1665 a959a8cc528f486a80a84e2ab233457870d255a1 normalizations.py
100644 blob 1402 85e8e32fc6b18fabf451d9290c9d33b95d453f5e normalizations.pyc
100644 blob 10925 a32ff93ffd1cfd5603eb93aa4bd63b921c93ee66 percolation.py
100644 blob 6411 e81344c0f958c1b9ef244749208d4c4f6ca91bd0 percolation.pyc
100644 blob 1445 5c06bec9674e666d2a5633804e1579df1acb6a9e ru_file_importer.py
100644 blob 1976 2ec49e610f1a124fa66df97f6fd6e6bb230797e4 ru_file_importer.pyc
100644 blob 16641 ee1f7cf9d0fabd7fb76787e8c375cafd9e5c9f35 ru_metric_calculator.py
100644 blob 11412 3d1104701e131696433e467a5bef30a589105865 ru_metric_calculator.pyc
100644 blob 2101 faf282d2fa893c507daa12ae97d29407238b3748 ru_start.py
100755 blob 2122 2fa307ee1f510eefc21af359b48003c3b368c85b start.py
100644 blob 2150 cf8423b852f3c09938906493ea831d0cfbbec941 statistics.py
100644 blob 2126 f8f55b53a1757681125d5af18c87980b44afe6bb statistics.pyc
100644 blob 1107 90c62617dd2c7340c2f3d31a3f485d30f9736865 test.py
100644 blob 6574 39e8f0848567e9d6836abb6283290ab35f5fdde3 visualization.py
100644 blob 5643 a4a569c1d73df7dec64e48b266a38bb481e0c0bb visualization.pyc
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/coria/coria-backend

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/coria/coria-backend

Clone this repository using git:
git clone git://git.rocketgit.com/user/coria/coria-backend

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main