Swe-related changes

refact-agent/engine/README.md

@@ -63,9 +63,9 @@ Installable by the end user:
 
 - [x] Code completion with RAG
 - [x] Chat with tool usage
-- [x] definition() references() tools
-- [x] vecdb search() with scope (semantic search)
-- [x] regex_search() with scope (pattern matching)
+- [x] search_symbol_definition() search_symbol_usages() tools
+- [x] search_semantic() with scope (semantic search)
+- [x] search_pattern() with scope (pattern matching)
 - [x] @file @tree @web @definition @references @search mentions in chat
 - [x] locate() uses test-time compute to find good project cross-section
 - [x] Latest gpt-4o gpt-4o-mini

refact-agent/engine/python_binding_and_cmdline/refact/chat_client.py

@@ -548,7 +548,10 @@ def print_messages(
 
     def con(x):
         if console:
-            console.print(x)
+            try:
+                console.print(x)
+            except:
+                print(x)
 
     def _is_tool_call(m: Message) -> bool:
         return m.tool_calls is not None and len(m.tool_calls) > 0

refact-agent/engine/src/ast/ast_db.rs

@@ -294,10 +294,15 @@ pub async fn doc_remove(ast_index: Arc<AMutex<AstDB>>, cpath: &String)
 }
 
 pub async fn doc_defs(ast_index: Arc<AMutex<AstDB>>, cpath: &String) -> Vec<Arc<AstDefinition>>
+{
+    let db = ast_index.lock().await.sleddb.clone();
+    doc_def_internal(db, cpath)
+}
+
+pub fn doc_def_internal(db: Arc<Db>, cpath: &String) -> Vec<Arc<AstDefinition>>
 {
     let to_search_prefix = filesystem_path_to_double_colon_path(cpath);
     let d_prefix = format!("d|{}::", to_search_prefix.join("::"));
-    let db = ast_index.lock().await.sleddb.clone();
     let mut defs = Vec::new();
     let mut iter = db.scan_prefix(d_prefix);
     while let Some(Ok((_, value))) = iter.next() {

refact-agent/engine/src/ast/ast_structs.rs

@@ -65,7 +65,7 @@ pub struct AstDB {
     pub ast_max_files: usize,
 }
 
-#[derive(Serialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct AstStatus {
     #[serde(skip)]
     pub astate_notify: Arc<ANotify>,

refact-agent/engine/src/at_commands/at_tree.rs

@@ -3,18 +3,16 @@ use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
 
 use async_trait::async_trait;
+use sled::Db;
 use tokio::sync::Mutex as AMutex;
 use tracing::warn;
 
-use crate::ast::ast_structs::AstDB;
-// use crate::ast::ast_indexer_thread::AstIndexService;
-// use crate::ast::treesitter::structs::SymbolType;
+use crate::ast::ast_structs::{AstDB, SymbolType};
 use crate::at_commands::at_commands::{AtCommand, AtCommandsContext, AtParam};
 use crate::at_commands::at_file::return_one_candidate_or_a_good_error;
 use crate::at_commands::execute_at::AtCommandMember;
 use crate::call_validation::{ChatMessage, ContextEnum};
 use crate::files_correction::{correct_to_nearest_dir_path, get_project_dirs, paths_from_anywhere};
-// use crate::files_in_workspace::Document;
 
 
 pub struct AtTree {
@@ -113,36 +111,27 @@ pub fn construct_tree_out_of_flat_list_of_paths(paths_from_anywhere: &Vec<PathBu
     root_nodes
 }
 
-fn _print_symbols(_entry: &PathsHolderNode) -> String
-{
-    // XXX fix tree
-    // if let Some(ast) = ast_index_maybe {
-    //     let doc = Document { doc_path: entry.path.clone(), doc_text: None };
-    //     match ast.get_by_file_path(RequestSymbolType::Declaration, &doc) {
-    //         Ok(symbols) => {
-    //             let symbols_list = symbols
-    //                 .iter()
-    //                 .filter(|x| x.symbol_type == SymbolType::StructDeclaration
-    //                     || x.symbol_type == SymbolType::FunctionDeclaration)
-    //                 .filter(|x| !x.name.is_empty() && !x.name.starts_with("anon-"))
-    //                 .map(|x| x.name.clone())
-    //                 .collect::<Vec<String>>()
-    //                 .join(", ");
-    //             if !symbols_list.is_empty() { format!(" ({symbols_list})") } else { "".to_string() }
-    //         }
-    //         Err(_) => "".to_string()
-    //     }
-    // } else {
-        "".to_string()
-    // }
+fn _print_symbols(db: Arc<Db>, entry: &PathsHolderNode) -> String {
+    let cpath = entry.path.to_string_lossy().to_string();
+    let defs = crate::ast::ast_db::doc_def_internal(db.clone(), &cpath);
+    let symbols_list = defs
+        .iter()
+        .filter(|x| match x.symbol_type {
+            SymbolType::StructDeclaration | SymbolType::TypeAlias | SymbolType::FunctionDeclaration => true,
+            _ => false
+        })
+        .map(|x| x.name())
+        .collect::<Vec<String>>()
+        .join(", ");
+    if !symbols_list.is_empty() { format!(" ({symbols_list})") } else { "".to_string() }
 }
 
-fn _print_files_tree(
+async fn _print_files_tree(
     tree: &Vec<PathsHolderNodeArc>,
     ast_db: Option<Arc<AMutex<AstDB>>>,
     maxdepth: usize,
 ) -> String {
-    fn traverse(node: &PathsHolderNodeArc, depth: usize, maxdepth: usize, ast_db: Option<Arc<AMutex<AstDB>>>) -> Option<String> {
+    fn traverse(node: &PathsHolderNodeArc, depth: usize, maxdepth: usize, db_mb: Option<Arc<Db>>) -> Option<String> {
         if depth > maxdepth {
             return None;
         }
@@ -151,14 +140,18 @@ fn _print_files_tree(
         let indent = "  ".repeat(depth);
         let name = if node.is_dir { format!("{}/", node.file_name()) } else { node.file_name() };
         if !node.is_dir {
-            output.push_str(&format!("{}{}{}\n", indent, name, _print_symbols(&node)));
+            if let Some(db) = &db_mb {
+                output.push_str(&format!("{}{}{}\n", indent, name, _print_symbols(db.clone(), &node)));
+            } else {
+                output.push_str(&format!("{}{}\n", indent, name));
+            }
             return Some(output);
         }
         output.push_str(&format!("{}{}\n", indent, name));
         let (mut dirs, mut files) = (0, 0);
         let mut child_output = String::new();
         for child in &node.child_paths {
-            if let Some(child_str) = traverse(child, depth + 1, maxdepth, ast_db.clone()) {
+            if let Some(child_str) = traverse(child, depth + 1, maxdepth, db_mb.clone()) {
                 child_output.push_str(&child_str);
             } else {
                 dirs += child.0.read().unwrap().is_dir as usize;
@@ -175,7 +168,12 @@ fn _print_files_tree(
 
     let mut result = String::new();
     for node in tree {
-        if let Some(output) = traverse(&node, 0, maxdepth, ast_db.clone()) {
+        let db_mb = if let Some(ast) = ast_db.clone() {
+            Some(ast.lock().await.sleddb.clone())
+        } else {
+            None
+        };
+        if let Some(output) = traverse(&node, 0, maxdepth, db_mb.clone()) {
             result.push_str(&output);
         } else {
             break;
@@ -184,20 +182,20 @@ fn _print_files_tree(
     result
 }
 
-fn _print_files_tree_with_budget(
+async fn _print_files_tree_with_budget(
     tree: Vec<PathsHolderNodeArc>,
     char_limit: usize,
     ast_db: Option<Arc<AMutex<AstDB>>>,
 ) -> String {
     let mut good_enough = String::new();
     for maxdepth in 1..20 {
-        let bigger_tree_str = _print_files_tree(&tree, ast_db.clone(), maxdepth);
+        let bigger_tree_str = _print_files_tree(&tree, ast_db.clone(), maxdepth).await;
         if bigger_tree_str.len() > char_limit {
             break;
         }
         good_enough = bigger_tree_str;
     }
-    return good_enough;
+    good_enough
 }
 
 pub async fn print_files_tree_with_budget(
@@ -218,10 +216,11 @@ pub async fn print_files_tree_with_budget(
     }
     match ast_module_option {
         Some(ast_module) => {
+            crate::ast::ast_indexer_thread::ast_indexer_block_until_finished(ast_module.clone(), 20_000, true).await;
             let ast_db: Option<Arc<AMutex<AstDB>>> = Some(ast_module.lock().await.ast_index.clone());
-            Ok(_print_files_tree_with_budget(tree, char_limit, ast_db.clone()))
+            Ok(_print_files_tree_with_budget(tree, char_limit, ast_db.clone()).await)
         }
-        None => Ok(_print_files_tree_with_budget(tree, char_limit, None)),
+        None => Ok(_print_files_tree_with_budget(tree, char_limit, None).await),
     }
 }
 
@@ -267,7 +266,6 @@ impl AtCommand for AtTree {
         };
 
         let use_ast = args.iter().any(|x| x.text == "--ast");
-
         let tree = print_files_tree_with_budget(ccx.clone(), tree, use_ast).await.map_err(|err| {
             warn!("{}", err);
             err

refact-agent/engine/src/file_filter.rs

@@ -3,7 +3,7 @@ use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::PathBuf;
 
-const LARGE_FILE_SIZE_THRESHOLD: u64 = 180*1024; // 180k files (180k is ~0.2% of all files on our dataset)
+const LARGE_FILE_SIZE_THRESHOLD: u64 = 4096*1024; // 4Mb files
 const SMALL_FILE_SIZE_THRESHOLD: u64 = 5;        // 5 Bytes
 
 pub const SOURCE_FILE_EXTENSIONS: &[&str] = &[

refact-agent/engine/src/forward_to_openai_endpoint.rs

@@ -49,9 +49,12 @@ pub async fn forward_to_openai_style_endpoint(
         data["temperature"] = serde_json::Value::from(temperature);
     }
     data["max_completion_tokens"] = serde_json::Value::from(sampling_parameters.max_new_tokens);
-    info!("NOT STREAMING TEMP {}", sampling_parameters.temperature
-        .map(|x| x.to_string())
-        .unwrap_or("None".to_string()));
+    info!("Request: model={}, reasoning_effort={}, T={}, n={}, stream=false", 
+        model_rec.name,
+        sampling_parameters.reasoning_effort.clone().map(|x| x.to_string()).unwrap_or("none".to_string()),
+        sampling_parameters.temperature.clone().map(|x| x.to_string()).unwrap_or("none".to_string()),
+        sampling_parameters.n.clone().map(|x| x.to_string()).unwrap_or("none".to_string())
+    );
     if is_passthrough {
         passthrough_messages_to_json(&mut data, prompt, &model_rec.name);
     } else {
@@ -133,9 +136,12 @@ pub async fn forward_to_openai_style_endpoint_streaming(
     }
     data["max_completion_tokens"] = serde_json::Value::from(sampling_parameters.max_new_tokens);
 
-    info!("STREAMING TEMP {}", sampling_parameters.temperature
-        .map(|x| x.to_string())
-        .unwrap_or("None".to_string()));
+    info!("Request: model={}, reasoning_effort={}, T={}, n={}, stream=true", 
+        model_rec.name,
+        sampling_parameters.reasoning_effort.clone().map(|x| x.to_string()).unwrap_or("none".to_string()),
+        sampling_parameters.temperature.clone().map(|x| x.to_string()).unwrap_or("none".to_string()),
+        sampling_parameters.n.clone().map(|x| x.to_string()).unwrap_or("none".to_string())
+    );
 
     if let Some(meta) = meta {
         data["meta"] = json!(meta);

refact-agent/engine/src/http/routers/v1/chat.rs

@@ -48,7 +48,8 @@ pub fn available_tools_by_chat_mode(current_tools: Vec<Value>, chat_mode: &ChatM
         ChatMode::NO_TOOLS => {
             vec![]
         },
-        ChatMode::EXPLORE | ChatMode::AGENT => current_tools,
+        ChatMode::AGENT => filter_out_tools(&current_tools, &vec!["search_symbol_definition", "search_symbol_usages", "search_pattern", "search_semantic"]),
+        ChatMode::EXPLORE => current_tools,
         ChatMode::CONFIGURE => filter_out_tools(&current_tools, &vec!["tree", "locate", "knowledge", "search"]),
         ChatMode::PROJECT_SUMMARY => keep_tools(&current_tools, &vec!["cat", "tree", "bash"]),
     }

refact-agent/engine/src/http/routers/v1/status.rs

@@ -1,20 +1,20 @@
 use axum::Extension;
 use axum::response::Result;
 use hyper::{Body, Response, StatusCode};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 use crate::ast::ast_structs::AstStatus;
 use crate::custom_error::ScratchError;
 use crate::global_context::SharedGlobalContext;
 
-#[derive(Serialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct RagStatus {
     pub ast: Option<AstStatus>,
-    ast_alive: String,
+    pub ast_alive: String,
     #[cfg(feature="vecdb")]
     pub vecdb: Option<crate::vecdb::vdb_structs::VecDbStatus>,
-    vecdb_alive: String,
-    vec_db_error: String,
+    pub vecdb_alive: String,
+    pub vec_db_error: String,
 }
 
 pub async fn get_rag_status(gcx: SharedGlobalContext) -> RagStatus {

refact-agent/engine/src/integrations/integr_pdb.rs

@@ -122,15 +122,20 @@ impl Tool for ToolPdb {
         if python_command.is_empty() {
             python_command = "python3".to_string();
         }
-        if command_args.windows(2).any(|w| w == ["-m", "pdb"]) {
+        if command_args.iter().any(|x| x.starts_with("python")) {
+            if !command_args.windows(2).any(|w| w == ["-m", "pdb"]) {
+                let python_index = command_args.iter().position(|x| x.starts_with("python")).ok_or("Open a new session by running pdb(\"python -m pdb my_script.py\")")?;
+                command_args.insert(python_index + 1, "pdb".to_string());
+                command_args.insert(python_index + 1, "-m".to_string());
+            }
             let output = start_pdb_session(&python_command, &mut command_args, &session_hashmap_key, &workdir_maybe, gcx.clone(), 10).await?;
             return Ok(tool_answer(output, tool_call_id));
         }
 
         let command_session = {
             let gcx_locked = gcx.read().await;
             gcx_locked.integration_sessions.get(&session_hashmap_key)
-                .ok_or("There is no active pdb session in this chat, you can open it by running pdb(\"python -m pdb my_script.py\")")?
+                .ok_or("There is no active pdb session in this chat. Open a new session by running pdb(\"python -m pdb my_script.py\")")?
                 .clone()
         };
 
@@ -139,7 +144,7 @@ impl Tool for ToolPdb {
             .ok_or("Failed to downcast to PdbSession")?;
 
         let output = match command_args[0].as_str() {
-            "kill" => {
+            "kill" | "q" | "quit" => {
                 let mut gcx_locked = gcx.write().await;
                 gcx_locked.integration_sessions.remove(&session_hashmap_key);
                 "Pdb session has been killed".to_string()

refact-agent/engine/src/known_models.json

@@ -432,7 +432,11 @@
             "scratchpad": "PASSTHROUGH",
             "tokenizer": "hf://Xenova/gpt-4o",
             "similar_models": [
-                "openai/o3-mini"
+                "openai/o3-mini",
+                "o3",
+                "openai/o3",
+                "o4-mini",
+                "openai/o4-mini"
             ]
         },      
         "o4-mini": {

refact-agent/engine/src/postprocessing/pp_context_files.rs

@@ -89,7 +89,8 @@ fn collect_lines_from_files(
         let file = lines.first().unwrap().file_ref.clone();
         if file.symbols_sorted_by_path_len.is_empty() {
             info!("{file_name} ignoring skeletonize because no symbols found in the file, maybe the file format is not supported or the file is empty");
-            lines.iter_mut().for_each(|x| x.take_ignoring_floor = true);
+            // Don't automatically mark all lines for inclusion when AST is unavailable
+            // This will be handled based on specific line range requests in convert_input_into_usefullness
         }
     }
 
@@ -316,10 +317,9 @@ async fn pp_limit_and_merge(
             anything = true;
             if first_line == 0 { first_line = i; }
             if i > prev_line + 1 {
-                out.push_str("...\n".to_string().as_str());
+                out.push_str("...\n");
             }
-            out.push_str(&line_ref.line_content);
-            out.push_str("\n");
+            out.push_str(&format!("{:4} | {}\n", line_ref.line_n + 1, line_ref.line_content));
             prev_line = i;
         }
         if last_line > prev_line + 1 {

refact-agent/engine/src/scratchpads/chat_utils_limit_history.rs

@@ -348,7 +348,7 @@ fn compress_duplicate_context_files(messages: &mut Vec<ChatMessage>) -> Result<(
             if !compressed_files.is_empty() {
                 let compressed_files_str = compressed_files.join(", ");
                 if remaining_files.is_empty() {
-                    let summary = format!("💿 Duplicate context file compressed: '{}' files were shown earlier in the conversation history", compressed_files_str);
+                    let summary = format!("💿 Duplicate files compressed: '{}' files were shown earlier in the conversation history. Do not ask for these files again.", compressed_files_str);
                     messages[file.msg_idx].content = ChatContent::SimpleText(summary);
                     messages[file.msg_idx].role = "cd_instruction".to_string();
                     tracing::info!("Stage 0: Fully compressed ContextFile at index {}: all {} files removed", 

refact-agent/engine/src/scratchpads/scratchpad_utils.rs

@@ -65,17 +65,11 @@ pub fn max_tokens_for_rag_chat_by_tools(
         };
 
         let tool_limit = match tool.function.name.as_str() {
-            "search" | "regex_search" | "definition" | "references" | "cat" if is_cat_with_lines => {
-                if context_files_len < crate::http::routers::v1::chat::CHAT_TOP_N {
-                    // Scale down proportionally to how much we exceed the context limit
-                    let scaling_factor = crate::http::routers::v1::chat::CHAT_TOP_N as f64 / context_files_len as f64;
-                    (4096.0 * scaling_factor) as usize
-                } else {
-                    4096
-                }
+            "search_semantic" | "search_pattern" | "search_symbol_definition" | "search_symbol_usages" | "cat" if is_cat_with_lines => {
+                (4096 * context_files_len).min(base_limit / 2).max(4096)
             },
-            "cat" | "locate" => 8192,
-            _ => 4096,  // Default limit for other tools
+            "cat" | "locate" => (8192 * context_files_len).min(base_limit / 2).max(8192),
+            _ => (4096 * context_files_len).min(base_limit / 2).max(4096)
         };
 
         overall_tool_limit += tool_limit;