Lots of fixes to support current versions of TF and Masscan

aws_tf/action_run.tpl

@@ -1,22 +1,68 @@
 #!/bin/bash 
-# This makes and runs masscan, use it for whatever you would like though. 
+# This makes and runs masscan, nmap, and whatwaf
 
 apt update -y 
 apt install git -y
 apt install build-essential -y
+apt install python3-pip -y
 snap install aws-cli --classic
-
-sudo apt-get install git gcc make libpcap-dev
+# pip install yq
+# apt install jq -y
+git clone https://github.com/EnableSecurity/wafw00f.git
+cd wafw00f
+python3 setup.py install
+cd /
+sudo apt-get install git gcc make libpcap-dev -y
 git clone https://github.com/robertdavidgraham/masscan
 cd masscan
 make -j
 
+apt install nmap -y
+# Format the date as 'YYYY-MM-DD'
+
+sudo git clone https://github.com/scipag/vulscan scipag_vulscan
+sudo ln -s `pwd`/scipag_vulscan /usr/share/nmap/scripts/vulscan
+
+
 /snap/bin/aws s3 cp s3://${s3_bucket}/${scan_list} .
 
-sudo bin/masscan --top-ports 50 -iL ${scan_list} --rate 500 --excludefile data/exclude.conf -oB results-${count}.masscan.bin --shard ${count}/${total}
+# do only port scan no banner grabbing, output to binary file
+#sudo bin/masscan --top-ports 50 -iL ${scan_list} --rate 500 --excludefile data/exclude.conf -oB results-${count}.masscan.bin --shard ${count}/${total} --seed 10
+
+# same, but output -oG masscan_results.txt, and run nmap on the results
+echo 'shards - ${count}/${total} '
+sudo bin/masscan --top-ports 50 -iL ${scan_list} --rate 500 --excludefile data/exclude.conf -oG masscan_results.txt --shard ${count}/${total} --seed 10
+awk '/open/ {split($7,a,"/"); print $4":"a[1]}' masscan_results.txt > nmap_targets.txt
+while IFS=":" read -r ip port; do
+
+temp_file="temp_nmap_$ip-$port.xml"
+result_file="nmap_results_$ip-$port.xml"
+
+# Perform the nmap scan and output to a temporary file
+sudo nmap -p $port -Pn -T4 --open --script http-headers,http-title --script-args http.useragent="A friendly web crawler (https://rescana.com)",http-headers.useget $ip -oX $temp_file
+
+# Check if grep finds 'http-headers', and if so, save to the final file
+if grep -q "output=" "$temp_file"; then # this pattern finds ips with http/s
+        echo "http://$ip:$port" >> whatwaf_targets.txt
+        echo "https://$ip:$port" >> whatwaf_targets.txt
+fi
+
+# Optionally, remove the temporary file
+rm "$temp_file"
+
+done < nmap_targets.txt
+
+# next step - run whatwaf on the results   
+#sh -c "yes no | whatwaf --skip -t 5 -F -C -l /masscan/whatwaf_targets.txt -o /masscan/vm${count}_wwres.csv" # didn't work due to tool prompting
+wafw00f -i whatwaf_targets.txt -o wafwoof.csv
 
-/snap/bin/aws s3 cp results-${count}.masscan.bin s3://${s3_bucket}/results-${count}.masscan.bin
+# upload results to s3 (txt) folder is date
+# for file in nmap_results_*.xml; do
+#     /snap/bin/aws s3 cp "$file" s3://${s3_bucket}/$(date +%F)/"$file"
 
+# done
 
+/snap/bin/aws s3 cp wafwoof.csv s3://${s3_bucket}/$(date +%F)/results-${count}-wafwoof.csv
+# upload results to s3 (bin)
+#/snap/bin/aws s3 cp results-${count}.masscan.bin s3://${s3_bucket}/results-${count}.masscan.bin
 
-

aws_tf/action_run_archive.tpl

@@ -0,0 +1,50 @@
+#!/bin/bash 
+# This makes and runs masscan, use it for whatever you would like though. 
+
+apt update -y 
+apt install git -y
+apt install build-essential -y
+apt install python-pip -y
+snap install aws-cli --classic
+# pip install yq
+# apt install jq -y
+sudo apt-get install git gcc make libpcap-dev -y
+git clone https://github.com/robertdavidgraham/masscan
+cd masscan
+make -j
+
+apt install nmap -y
+# Format the date as 'YYYY-MM-DD'
+
+sudo git clone https://github.com/scipag/vulscan scipag_vulscan
+sudo ln -s `pwd`/scipag_vulscan /usr/share/nmap/scripts/vulscan
+
+
+/snap/bin/aws s3 cp s3://${s3_bucket}/${scan_list} .
+
+# do only port scan no banner grabbing, output to binary file
+#sudo bin/masscan --top-ports 50 -iL ${scan_list} --rate 500 --excludefile data/exclude.conf -oB results-${count}.masscan.bin --shard ${count}/${total} --seed 10
+
+# same, but output -oG masscan_results.txt, and run nmap on the results
+sudo bin/masscan --top-ports 50 -iL ${scan_list} --rate 500 --excludefile data/exclude.conf -oG masscan_results.txt --shard ${count}/${total} --seed 10
+awk '/open/ {split($7,a,"/"); print $4":"a[1]}' masscan_results.txt > nmap_targets.txt
+while IFS=: read -r ip port; do
+# for banner grabbing + passive vuln detection
+#    sudo nmap -sV -p $port $ip --script=vulscan/vulscan.nse --script-args vulscandb=cve.csv -P0 -oX "nmap_results_$ip.xml"
+
+# for banner grabbing only (for rescana)
+# nmap -sV -p80 -P0 -script=banner 188.114.97.7 -oX test.xml
+sudo nmap -sV -p $port -P0 -script=banner $ip -oX "nmap_results_$ip.xml"
+done < nmap_targets.txt
+
+# upload results to s3 (txt) folder is date
+for file in nmap_results_*.xml; do
+    /snap/bin/aws s3 cp "$file" s3://${s3_bucket}/$(date +%F)/"$file"
+done
+
+# upload results to s3 (bin)
+#/snap/bin/aws s3 cp results-${count}.masscan.bin s3://${s3_bucket}/results-${count}.masscan.bin
+
+
+
+

aws_tf/main.tf

@@ -76,29 +76,45 @@ resource "aws_instance" "vm-ubuntu" {
   iam_instance_profile = "${aws_iam_instance_profile.temp_profile.name}"
   count         = "${var.instance_count}"
   user_data = "${element(data.template_file.init.*.rendered, count.index)}"
-  ami                         = "ami-04b9e92b5572fa0d1"
+  ami                         = "ami-0fc5d935ebf8bc3bc" # now ubuntu 22.04   # was "ami-04b9e92b5572fa0d1"
   instance_type               = "t2.micro"
   key_name                    = "temp_key"
   associate_public_ip_address = true
   source_dest_check           = false
   vpc_security_group_ids      = [aws_security_group.sg-ubuntu.id]
+  depends_on = [
+  aws_key_pair.generated_key
+  ]
 }
 
 resource "random_id" "s3" {
   byte_length = 8
 }
 
 resource "aws_s3_bucket" "scanning_storage" {
-  bucket = "${random_id.s3.hex}"
+  bucket        = random_id.s3.hex
   force_destroy = true
-  acl    = "private"
 }
-resource "aws_s3_bucket_object" "object" {
-  bucket = "${random_id.s3.hex}"
-  key    = "${var.scan_list}"
-  source = "${var.scan_list}"
-  depends_on = [aws_s3_bucket.scanning_storage]
+
+
+
+resource "aws_s3_bucket_ownership_controls" "scanning_storage_ownership" {
+  bucket = aws_s3_bucket.scanning_storage.id
+  rule {
+    object_ownership = "BucketOwnerPreferred"
+  }
 }
+
+resource "aws_s3_object" "object" {
+  bucket = aws_s3_bucket.scanning_storage.id
+  key    = var.scan_list
+  source = var.scan_list
+  depends_on = [
+    aws_s3_bucket.scanning_storage,
+    aws_s3_bucket_ownership_controls.scanning_storage_ownership
+  ]
+}
+
 # UBUNTU SECURITY GROUP
 resource "aws_security_group" "sg-ubuntu" {
   name        = "sg_${var.host_name}"

aws_tf/outputs.tf

@@ -12,6 +12,7 @@ output "instance_private_ip_addresses" {
 }
 output "Private_SSH_Key_Value" {
   value = "${tls_private_key.temp_key.private_key_pem}"
+  sensitive = true
 }
 
 output "s3_bucket" {

fix.py

@@ -0,0 +1,53 @@
+from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent
+from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent
+
+llm_config = {
+    "request_timeout": 600,
+    "config_list": config_list,
+    "temperature": 0
+}
+
+
+interface = autogen_interface.AutoGenInterface()
+persistence_manager=InMemoryStateManager()
+persona = "I am a strategic analyst, trained in cyber security. I worked for the NSA."
+human = "Im a team manager at this company"
+memgpt_agent=presets.use_preset(presets.DEFAULT_PRESET, model='gpt-4', persona=persona, human=human, interface=interface, persistence_manager=persistence_manager, agent_config=llm_config)
+
+
+if not USE_MEMGPT:
+    # In the AutoGen example, we create an AssistantAgent to play the role of the coder
+    assistant = RetrieveAssistantAgent(
+        name="assistant",
+        system_message="You are a helpful assistant.",
+        human_input_mode="TERMINATE",
+        llm_config=llm_config,
+        human_input_mode="TERMINATE",
+    )
+
+else:
+    # In our example, we swap this AutoGen agent with a MemGPT agent
+    # This MemGPT agent will have all the benefits of MemGPT, ie persistent memory, etc.
+    print("\nMemGPT Agent at work\n")
+    coder = memgpt_autogen.MemGPTAgent(
+        name="MemGPT_assistant",
+        agent=memgpt_agent,
+    )
+
+
+
+
+
+
+rag_agent = RetrieveUserProxyAgent(
+    human_input_mode="NEVER",
+    retrieve_config={
+        "task": "qa",
+        "docs_path": "/content/drive/MyDrive/ColabDataSets/shaydocs",
+        "collection_name": "rag_collection",
+        "embedding_function": openai_embedding_function,
+        "custom_text_split_function": text_splitter.split_text,
+        "get_or_create": True,
+        "max_consecutive_auto_reply": 10
+    },
+)