Python General Notes.txt

##
## https://github.com/python/mypy
##

# writing dependencies to a file
sudo pip install -r confs/pip/commons.txt
....................
# comment
<PACKAGE>
<PACKAGE>==<VERSION>
....................

# query package versions in pip
sudo pip install <PACKAGE>==

# pip uninstall package
sudo pip uninstall <PACKAGE>

# use argv
import sys
sys.argv[0] -> empty, -c or fullname of the located module

# if
the_world_is_flat = True
if the_world_is_flat:
	print("Be careful not to fall off!")

# are comments
# e.g.

# Hi i'm a comment

# Note
text = "# This is not a comment because it's inside quotes."

# IMPORTANT NOTE!
Division (/) always returns a float. 
17 / 3  # classic division returns a float
17 // 3  # floor division discards the fractional part

# power operator
>>> 5 ** 2  # 5 squared
25

# complex numbers in python
3+5j

# strings
'spam eggs'  # single quotes
"doesn't"  # ...or use double quotes instead
r'C:\some\name' # raw strings, note the r before the quote

# multiline strings
print("""\
Usage: thingy [OPTIONS]
     -h                        Display this usage message
     -H hostname               Hostname to connect to
""")

# Strings can be concatenated (glued together) with the + operator, and repeated with *:
>>> # 3 times 'un', followed by 'ium'
>>> 3 * 'un' + 'ium'

# Two or more string literals (i.e. the ones enclosed between quotes) next to each other are automatically concatenated.
>>> 'Py' 'thon'
>>> text = ('Put several strings within parentheses '
...         'to have them joined together.')

# Strings can be indexed (subscripted), with the first character having index 0. There is no separate character type; a character is simply a string of size one:
>>> word = 'Python'
>>> word[0]  # character in position 0
'P'
>>> word[5]  # character in position 5
'n'
>>> word[-1]  # last character
'n'
>>> word[-2]  # second-last character
'o'
>>> word[-6]
'P'
# formatting strings
'this {0} is like {1}'.format(a,b)

# slicing
word[0:2]
word[:2] + word[2:]

# IMPORTANT!
# Operator [] does not work for out of range, but works for slices!

# string length
>>> s = 'hello'
>>> len(s)

# type conversion
float('NaN')
int('314')

# replace
'str'.replace('a','b')

# define byte array from string literal!
p = b'hello, world!'

# comparisions
 -> in, not in -> a value occurs in sequence
 -> is, not is -> two objects are same
 -> chaining -> a < b == c
 -> boolean: and or not
 -> =, !=
 -> if x -> if x is not None
 -> lexographic on sequences (TypeError is not same sequence type)

# lists
>>> squares = [1, 4, 9, 16, 25]
# shallow copy
>>> squares[:]
# shallow copy another method
list.copy()
# concatenation
>>> squares + [36, 49, 64, 81, 100]
# append
cubes.append(216)
# append another method
a[len(a):] = [x]
# insert at
words.insert(0, w)
# splice remove
>>> letters[2:5] = []
# splice remove another method
del a[2:4]
# clear
letters[:] = []
# clear another method
list.clear()
# clear yet another method
del list[:]
# sorting
pairs.sort(key=lambda pair: pair[1])
# sorting (more)
list.sort(key=None, reverse=False)
# extend list
list.extend(iterable)
# extend list another method
a[len(a):] = iterable
# remove first occurance of item
# IT IS ERROR IF THERE IS NO SUCH ITEM!
list.remove(x)
# remove last element and RETURN IT!
removedItem = list.pop()
# remove item at specific index and RETURN IT!
removedItem = list.pop(index)
# find index of item
# IT IS "ValueError" IF ITEM IS NOT FOUND!
list.index(x[, start[, end]])
# find number of times an item appears in list
list.count(x)
# reverse a list
list.reverse()
# remove index from list
del list[0]

## deque (doubly ended queue)
from collections import deque
queue = deque(["Eric", "John", "Michael"])
queue.append("Terry")
queue.popleft()

## tuple
## IMPORTANT! TUPLES ARE IMMUTABLE!
>>> t = 12345, 54321, 'hello!'
>>> t[0]
>>> empty = ()
>>> singleton = 'hello',
>>> len(t)
## tuple packing
t = 12345, 54321, 'hello!'
## tuple unpacking
>>> x, y, z = t

## sets
# create
>>> s = {1, 2, 3}
# create via sequences
>>> s = set(range(3))
# membership
>>> 5 in s
# in a not in b
>>> a - b
# in a or in b or in both
>>> a | b
# in a and in b
>>> a & b
# in a and in b but not both
>>> a ^ b
# comprehension
>>> {x for x in 'abcdefg' if not x in 'abc'}

## dictionaries
# key?
-> string
-> number
-> tuple of
	-> string
	-> number
	-> tuple
# create
>>> tel = {'jack': 4098, 'sape': 4139}
# create another method
>>> dict([('sape', 4139), ('guido', 4127), ('jack', 4098)])
# assignment
a[key] = value
# delete key
del a[key]
# sequence of keys
a.keys()
# sequence of (key,value) tuple
a.items()
# looping
for k, v in dictionary.items():
 ...

# multiple assignment
a, b = 0, 1

# while
while a < 10:
	<DO>

# if
if x < 0:
 ...
elif x == 0:
 ...
else:
 ...

# if with "in" operator
if ok in ('y', 'ye', 'yes'):

# foreach
for w in myList:
for i in range(5): #0,1,2,3,4
range(5,10)
range(0,10,3)
 ...

# ATTENTION!
# else: on for loops
#  and a loop’s else clause runs when no break occurs.
...........................
>>> for n in range(2, 10):
...     for x in range(2, n):
...         if n % x == 0:
...             print(n, 'equals', x, '*', n//x)
...             break
...     else:
...         # loop fell through without finding a factor
...         print(n, 'is a prime number')
...........................

# creating a new list from iterable
>>> list(range(5))
[0, 1, 2, 3, 4]

# nop pass statement
......................................................
>>> while True:
...     pass

>>> class MyEmptyClass:
...     pass

>>> def initlog(*args):
...     pass   # Remember to implement this!
......................................................

# defining functions
def fib(n):
  ....

# function as values
def func():
 ...

f = func
f()

# ATTENTION!
# python has "None" for void
# much like JS it is referencable!
# in this sense python does not have procedures,
# but it has functions that return None
# much like empty return statement which is shortened for
# return None

# raise ValueError
raise ValueError('invalid user response')

# default arguements for functions
def ask_ok(prompt, retries=4, reminder='Please try again!'):

i = 5
def f(arg=i):
    print(arg)

def f(a, L=[]):
    L.append(a)
    return L

# keyword arguments
# passing named arguments
parrot(action='VOOOOOM', voltage=1000000)

# keywords continued
def cheeseshop(kind, *arguments, **keywords):
..................
cheeseshop("Limburger", "It's very runny, sir.",
           "It's really very, VERY runny, sir.",
           shopkeeper="Michael Palin",
           client="John Cleese",
           sketch="Cheese Shop Sketch")

# unpacking argument list
>>> args = [3, 6]
>>> list(range(*args)) 

>>> d = {"voltage": "four million", "state": "bleedin' demised", "action": "VOOM"}
>>> parrot(**d)

# tuples
x = (1, 'one')
x[0]
x[1]

# lambdas
y = lambda x: x + n

# docstring
>>> def my_function():
...     """Do nothing, but document it.
...
...     No, really, it doesn't do anything.
...     """
...     pass

# print docstring at runtime
>>> print(my_function.__doc__)

# annotations
# -> parameter annotations
# -> return annotations
# __annotations__ attribute
>>> def f(ham: str, eggs: str = 'eggs') -> str:
...     print("Annotations:", f.__annotations__)
...     print("Arguments:", ham, eggs)
...     return ham + ' and ' + eggs
...
>>> f('spam')
Annotations: {'ham': <class 'str'>, 'return': <class 'str'>, 'eggs': <class 'str'>}

# coding style considerations
# Every Python developer should read it at some point
# the convention is to use CamelCase for classes and lower_case_with_underscores for functions and methods.
# Always use self as the name for the first method argument
https://www.python.org/dev/peps/pep-0008/

### MAP, FILTER AND REDUCE! ###

# map
>>> list(map(lambda x: x **2, range(1,6)))
[1, 4, 9, 16, 25]

# filter
>>> list( filter((lambda x: x < 0), range(-5,5)))
[-5, -4, -3, -2, -1]

# zip
>>> list(zip([1,2],[3,4]))
[(1, 3), (2, 4)]

# sort
>>> sorted('dbca')
['a', 'b', 'c', 'd']

# make indexed
# enumerate
>>> list(enumerate('abc'))
[(0, 'a'), (1, 'b'), (2, 'c')]

# reversed
>>> list(reversed(range(3))

# max
max(...)

# min
min(...)

# sum
sum(...)

## See more at
## https://docs.python.org/3/library/itertools.html
## containing accumulate, zip and other cool stuff!

###############################

### Comprehension ###

[x**2 for x in range(10)]

[(x,y) for x in [1,2,3] for y in [3,1,4] if x != y]

# set
{x for x in 'abcdefg' if not x in 'abc'}

# dictionary
{x: x**2 for x in (2, 4, 6)}

###############################

# modules
-> REMEMBER INITIALIZATION!
-> REMEMBER "_" at the beginning of symbol is for private!
import <MODULE>
from <MODULE> import <SYMBOL>
from <MODULE> import *

--> __dict__ is the dictionary in each module's namespace

-> ImportError

# reload (to reload module initialization)
importlib.reload()
importlib.reload(modulename)

# REMEMBER!
import <MODULE>
dir(MODULE) -> lists all module functions
help(MODULE) -> returns manual page for module

# checking if module is launched as script
if __name__ == "__main__":
    import sys
    fib(int(sys.argv[1]))

-> sys.path
-> dir(module) -> array of defined names in a module
-> parent dir.. parent dirs!
-> dir() -> array of names defined in current module
-> dir(builtins)

# packages
 -> __init__.py
   -> can be empty
   -> can contain initialization code for package
   -> __all__ = [...] explicitly indicate which submodules to load for import *
   -> __path__ -> list containing the name of the directory holding the package's __init__.py
 -> sub-package

###############################

sys.stdout --> standard file for output

str(v) -> convert to string human readable
repr(v) -> convert to string parseable by interpreter

"xxx".rjust(2) -> width spaces to right (optional fill character as second argument)
"xxx".ljust(2) -> width spaces to left (optional fill character as second argument)
"xxx".center(2) -> width spaces to keep centered (optional fill character as second argument)

>>> for x in range(1, 11):
...     print('{0:2d} {1:3d} {2:4d}'.format(x, x*x, x*x*x))


>>> '12'.zfill(5)
'00012'

>>> print('We are the {} who say "{}!"'.format('knights', 'Ni'))
We are the knights who say "Ni!"

>>> print('{0} and {1}'.format('spam', 'eggs'))
spam and eggs

>>> print('This {food} is {adjective}.'.format(
...       food='spam', adjective='absolutely horrible'))

..............
{!a} ascii() {!s} str() {!r} repr()
{0} {} {named}
{0:10} {0:10d} {0:.3f}
{0[key]:d} given dictionary table ----> also can be done using ** operator
..............

% FORMATTING OPERATOR IS THE OLD STYLE!

>>> import math
>>> print('The value of PI is approximately %5.3f.' % math.pi)

## Opening file
>>> f = open('workfile', 'w')
w, r, a, r+, b

>>> with open('workfile') as f:
...     read_data = f.read()
>>> f.closed
True

f.read() -> total file
f.read(size) -> returns empty string '' if EOF
f.readLine() -> returns '\n' for empty lines, empty string '' if EOF
f.write(str) -> returns number of characters written
f.tell()
f.seek(offset, from_what) -> 0: beginning, 1: current, 2: end of file

>>> for line in f:
...     print(line, end='')

## JSON!

>>> import json
>>> json.dumps([1, 'simple', 'list'])
'[1, "simple", "list"]'
json.dump(x, f)
x = json.load(f)

REMEMBER: !!!!!!!!
pickle -> them module to work with serialization/deserialization of python specific objects!
REMEMBER: !!!!!!!!

EXCEPTIONS!

ZeroDivisionError: division by zero
NameError: name 'spam' is not defined
TypeError: Can't convert 'int' object tValueErroro str implicitly
ValueError:
RuntimeError:
OSError:
StopIteration: -> for iterators
KeyError: -> for dictionaries

try:
 ....
except ValueError:
 ....

except (RuntimeError, TypeError, NameError):
 pass

except:
    print("Unexpected error:", sys.exc_info()[0])
    raise

except OSError as err:

class B(Exception):
    pass

raise ValueError()

for arg in sys.argv[1:]:
    try:
        f = open(arg, 'r')
    except OSError:
        print('cannot open', arg)
    else:
        print(arg, 'has', len(f.readlines()), 'lines')
        f.close()

raise Exception('spam', 'eggs')
... except Exception as inst:
...     print(inst.args)     # arguments stored in .args
...     x, y = inst.args     # unpack args

raise ValueError  # shorthand for 'raise ValueError()'

>>> def divide(x, y):
...     try:
...         result = x / y
...     except ZeroDivisionError:
...         print("division by zero!")
...     else:
...         print("result is", result)
...     finally:
...         print("executing finally clause")

---> nonlocal variable definition:
nonlocal a
a = "hello"

---> global variable definition:
global a
a = "hello"

########### CLASSES ! #################

-> Class objects support two kinds of operations: attribute references and instantiation.

class MyClass:
    """A simple example class"""
    i = 12345

    def f(self):
        return 'hello world'

-> then MyClass.i and MyClass.f are valid attribute references, returning an integer and a function object, respectively.
-> NOTE:! class variables are shared by all instances

x = MyClass()

def __init__(self):
    self.data = []

-> Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. 

-> method objects
xf = x.f
while True:
    print(xf())

# inheritance
class DerivedClassName(BaseClassName):

# python 2.7
class ChildB(Base):
    def __init__(self):
        super(ChildB, self).__init__()

# python 3
class ChildB(Base):
    def __init__(self):
        super().__init__()

# without using super
 Base.__init__(self) # Avoid this.

# isinstance()
# issubclass()

# multiple inheritance
class DerivedClassName(Base1, Base2, Base3):

-> rember: dynamic ordering, left to right

# private
_score

# name mangling to prevent accidents
# of overrides
class Mapping:
    def __init__(self, iterable):
        self.items_list = []
        self.__update(iterable)

    def update(self, iterable):
        for item in iterable:
            self.items_list.append(item)

    __update = update   # private copy of original update() method


---> instance methods
__self__
__func__

## NOTE
## NOTE
## Behind the scenes, the for statement calls iter() on the container object. 

# create iter object manually
it = iter(s)
next(it)

# custom iterator!
# __iter__ and __next__ methods
# StopIteration exception
class Reverse:
    """Iterator for looping over a sequence backwards."""
    def __init__(self, data):
        self.data = data
        self.index = len(data)

    def __iter__(self):
        return self

    def __next__(self):
        if self.index == 0:
            raise StopIteration
        self.index = self.index - 1
        return self.data[self.index]


# generators!


def reverse(data):
    for index in range(len(data)-1, -1, -1):
        yield data[index]

>>> for char in reverse('golf'):
...     print(char)

# generator expressions
sum(i*i for i in range(10))
max((student.gpa, student.name) for student in graduates)
sum(x*y for x,y in zip(xvec, yvec))

######## REMEMBERRRRRRRRRR! SPECIAL METHODS DEFINED ON CLASS ##############
__init__
__str__
__repr__
__iter__
__len__

###########################################
os module
###########################################

import os
os.getcwd()      # Return the current working directory
os.chdir('/server/accesslogs')   # Change current working directory
os.system('mkdir today')   # Run the command mkdir in the system shell

import shutil
shutil.copyfile(source, dest)
shutil.move(source, dest)

# file wildcards
import glob
glob.glob('*.py') -> array of file names (relative)
-> better argument processing -> getopt module
-> better better argument processing -> argparser module

# argv
import sys
sys.argv -> array of arguments ([0] is main file)

# error redirection
sys.stderr.write(...)
sys.exit()

# regular expression
import re
re.findall(r'<REGEX>', str) -> array of matches
re.sub(r'<REGEX>', str) -> substituted string
re.match(r'<REGEX>', str) -> first match or None
re.finditer(r'<REGEX>', str) -> iterator to matches

# mathematics
import math
math.pi
math.cos()
math.sin()
math.log()

# random
import random
random.choice([1, 2, 3])
random.sample(range(100), 10)
random.random() -> random float
random.randrange(6) -> random integer

# statistics
import statistics
statistics.mean([1,2,3])
statistics.median([1,2,3])
statistics.variance([1,2,3])

# request url
from urllib.request import urlopen
with urlopen('URL') as response:
	for line in response:

# send main
import smtplib
server = smtplib.SMTP('localhost')
server.sendmail('TO', 'FROM',
"""To: <>
From: <>

Message!
""")
server.quit()

# date/time
from datetime import date
now = datetime.now()
datetime.date(2003, 12, 2)
>>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.")
'12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.'
dt = d1 - d2
dt.days

# compression
-> zlib, gzip, bz2, lzma, zipfile and tarfile modules!
import zlib
s = b'hey!'
t = zlib.compress(s)
zlib.decompress(t)
zlib.crc32(s)

# performance measurement!
-> timeit
-> profile
-> pstats

from timeit import Timer
Timer('<eval expression>').timeit()

# doctest! test through pydocs
def average(values):
    """Computes the arithmetic mean of a list of numbers.

    >>> print(average([20, 30, 70]))
    40.0
    """
    return sum(values) / len(values)

import doctest
doctest.testmod()   # automatically validate the embedded tests

# write unit tests
import unittest

class TestStatisticalFunctions(unittest.TestCase):

    def test_average(self):
        self.assertEqual(average([20, 30, 70]), 40.0)
        self.assertEqual(round(average([1, 5, 7]), 1), 4.3)
        with self.assertRaises(ZeroDivisionError):
            average([])
        with self.assertRaises(TypeError):
            average(20, 30, 70)

unittest.main()  # Calling from the command line invokes all tests

# output formatting!
import reprlib
reprlib.repr(...)

import pprint
pprint.pprint(x, width=30)

import textwrap
textwrap.fill(doc, width=30)

# binary data record layouts
import struct
fields = struct.unpack('<IIIHH', data)
< means little endian
I means 4 byte unsigned integer
H means 2 byte unsigned integer

# multithreading
import threading
class AsyncTask(threading.Thread)
  def __init__(self):
    threading.Thread.__init__(self)

  def run(self):
    ...

t = AsyncTask()
t.start()
t.join()

# logging
import logging
logging.[debug|info|warning|error|critical](...)

# weak referencing
import weakref
d = weakref.WeakValueDictionary()
d['key'] = obj
d['key'] -> KeyError

# compact arrays
>>> from array import array
>>> a = array('H', [4000, 10, 700, 22222])

# heap
>>> from heapq import heapify, heappop, heappush
>>> data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
>>> heapify(data)                      # rearrange the list into heap order
>>> heappush(data, -5)                 # add a new entry
>>> [heappop(data) for i in range(3)]  # fetch the three smallest entries

# decimal module
>>> from decimal import *
>>> round(Decimal('0.70') * Decimal('1.05'), 2)

# control precision
>>> getcontext().prec = 36
>>> Decimal(1) / Decimal(7)
Decimal('0.142857142857142857142857142857142857')

# staticmethods
# NO self!
@staticmethod
def dosomething(a, b, c)

# abstract class
from abc import ABC, abstractmethod
 
class AbstractClassExample(ABC):
 
    def __init__(self, value):
        self.value = value
        super().__init__()
    
    @abstractmethod
    def do_something(self):
        pass

# common header format (for __init__.py)
# "Prototype", "Development", or "Production"

__author__ = "Rob Knight, Gavin Huttley, and Peter Maxwell"
__copyright__ = "Copyright 2007, The Cogent Project"
__credits__ = ["Rob Knight", "Peter Maxwell", "Gavin Huttley",
                    "Matthew Wakefield"]
__license__ = "GPL"
__version__ = "1.0.1"
__maintainer__ = "Rob Knight"
__email__ = "rob@spot.colorado.edu"
__status__ = "Production"

# custom iterator protocol
class _Pipe(object):

    def __init__(self):
        self._condition = threading.Condition()
        self._values = []
        self._open = True

    def __iter__(self):
        return self

    def _next(self):
        with self._condition:
            while True:
                if self._values:
                    return self._values.pop(0)
                elif not self._open:
                    raise StopIteration()
                else:
                    self._condition.wait()

    def __next__(self):  # (Python 3 Iterator Protocol)
        return self._next()

    def next(self):  # (Python 2 Iterator Protocol)
        return self._next()


##########################################################33
pipenv
##########################################################33

# install pipenv
pip install --user pipenv

# create a new project on python2
pipenv --python 2
# or be more specific about version
pipenv --python 2.7.14
# or python 3
pipenv --python 3
# or (PREFERRED)
pipenv --three

pipenv install <PACKAGE>
pipenv install <PACKAGE> --dev
pipenv install
pipenv install --dev
pipenv shell <-- activate venv
pipenv run <FILE> <-- execute file in venv

set required python version
--------------------------
[requires]
python_version = "3.2"
--------------------------

# specify a package for windows only!

[packages]
requests = "*"
pywinusb = {version = "*", os_name = "== 'windows'"}

# specify package versions
==2.1.3
>=2.1.3
>=2.1.3,<3.0

#############################################

interactive environments

ipython -> jupyter notebook
bpython

pip install bpython
OR
apt-get install bpython

##########################################################33
Manual venv (python 2.x)
##########################################################33

pip install --user virtualenv
mkdir -p ~/.local/share/virtualenvs/<PROJECT>
virtualenv -p $(which python2) ~/.local/share/virtualenvs/<PROJECT>

# activate
source ~/.local/share/virtualenvs/<PROJECT>/bin/activate

# deactivate
deactivate

# delete venv
rm -rfv ~/.local/share/virtualenvs/<PROJECT>