Roll Your Own Salesforce "Lookup" Popup Window blog post

CustomAccountLookup.page

<apex:page controller="CustomAccountLookupController"
  title="Search" 
  showHeader="false" 
  sideBar="false" 
  tabStyle="Account" 
  id="pg">
  
  <apex:form >
  <apex:outputPanel id="page" layout="block" style="margin:5px;padding:10px;padding-top:2px;">
		<apex:tabPanel switchType="client" selectedTab="name1" id="tabbedPanel">
		
		  <!-- SEARCH TAB -->
      <apex:tab label="Search" name="tab1" id="tabOne">
		      
		    <apex:actionRegion >  
				  <apex:outputPanel id="top" layout="block" style="margin:5px;padding:10px;padding-top:2px;">
				    <apex:outputLabel value="Search" style="font-weight:Bold;padding-right:10px;" for="txtSearch"/>
				    <apex:inputText id="txtSearch" value="{!searchString}" />
				      <span style="padding-left:5px"><apex:commandButton id="btnGo" value="Go" action="{!Search}" rerender="searchResults"></apex:commandButton></span>
				  </apex:outputPanel>
	   
			    <apex:outputPanel id="pnlSearchResults" style="margin:10px;height:350px;overflow-Y:auto;" layout="block">
				    <apex:pageBlock id="searchResults"> 
				      <apex:pageBlockTable value="{!results}" var="a" id="tblResults">
				        <apex:column >
				          <apex:facet name="header">
				            <apex:outputPanel >Name</apex:outputPanel>
				          </apex:facet>
				           <apex:outputLink value="javascript:top.window.opener.lookupPick2('{!FormTag}','{!TextBox}_lkid','{!TextBox}','{!a.Id}','{!a.Name}', false)" rendered="{!NOT(ISNULL(a.Id))}">{!a.Name}</apex:outputLink>     
				        </apex:column>
				      </apex:pageBlockTable>
				    </apex:pageBlock>
			    </apex:outputPanel>
		    </apex:actionRegion>
		      
		  </apex:tab>
		  
		  <!-- NEW ACCOUNT TAB -->
		  <apex:tab label="New Account" name="tab2" id="tabTwo">

        <apex:pageBlock id="newAccount" title="New Account" >
        
          <apex:pageBlockButtons >
            <apex:commandButton action="{!saveAccount}" value="Save"/>
          </apex:pageBlockButtons>
          <apex:pageMessages />
        
          <apex:pageBlockSection columns="2">
            <apex:repeat value="{!$ObjectType.Account.FieldSets.CustomAccountLookup}" var="f">
              <apex:inputField value="{!Account[f]}"/>
            </apex:repeat>
          </apex:pageBlockSection> 
        </apex:pageBlock>
		      
		  </apex:tab>
		</apex:tabPanel>
  </apex:outputPanel>
  </apex:form>
</apex:page>

CustomAccountLookupController.cls

public with sharing class CustomAccountLookupController {
	
  public Account account {get;set;} // new account to create
  public List<Account> results{get;set;} // search results
  public string searchString{get;set;} // search keyword
  
  public CustomAccountLookupController() {
  	account = new Account();
  	// get the current search string
  	searchString = System.currentPageReference().getParameters().get('lksrch');
  	runSearch();  
  }
   
  // performs the keyword search
  public PageReference search() {
    runSearch();
    return null;
  }
  
  // prepare the query and issue the search command
  private void runSearch() {
  	// TODO prepare query string for complex serarches & prevent injections
    results = performSearch(searchString);               
  } 
  
  // run the search and return the records found. 
  private List<Account> performSearch(string searchString) {

    String soql = 'select id, name from account';
    if(searchString != '' && searchString != null)
      soql = soql +  ' where name LIKE \'%' + searchString +'%\'';
    soql = soql + ' limit 25';
    System.debug(soql);
    return database.query(soql); 

  }
  
  // save the new account record
  public PageReference saveAccount() {
    insert account;
    // reset the account
    account = new Account();
    return null;
  }
  
  // used by the visualforce page to send the link to the right dom element
  public string getFormTag() {
    return System.currentPageReference().getParameters().get('frm');
  }
    
  // used by the visualforce page to send the link to the right dom element for the text box
  public string getTextBox() {
    return System.currentPageReference().getParameters().get('txt');
  }
 
}

MyCustomLookup.page

<apex:page controller="MyCustomLookupController" id="Page" tabstyle="Contact">

  <script type="text/javascript"> 
  function openLookup(baseURL, width, modified, searchParam){
    var originalbaseURL = baseURL;
    var originalwidth = width;
    var originalmodified = modified;
    var originalsearchParam = searchParam;
    
    var lookupType = baseURL.substr(baseURL.length-3, 3);
    if (modified == '1') baseURL = baseURL + searchParam;
    
    var isCustomLookup = false;
    
    // Following "001" is the lookup type for Account object so change this as per your standard or custom object
    if(lookupType == "001"){
  
      var urlArr = baseURL.split("&");
      var txtId = '';
      if(urlArr.length > 2) {
        urlArr = urlArr[1].split('=');
        txtId = urlArr[1];
      }
      
      // Following is the url of Custom Lookup page. You need to change that accordingly
      baseURL = "/apex/CustomAccountLookup?txt=" + txtId;
      
      // Following is the id of apex:form control "myForm". You need to change that accordingly
      baseURL = baseURL + "&frm=" + escapeUTF("{!$Component.myForm}");
      if (modified == '1') {
        baseURL = baseURL + "&lksearch=" + searchParam;
      }
      
      // Following is the ID of inputField that is the lookup to be customized as custom lookup
      if(txtId.indexOf('Account') > -1 ){
        isCustomLookup = true;
      }
    }
    
    
    if(isCustomLookup == true){
      openPopup(baseURL, "lookup", 350, 480, "width="+width+",height=480,toolbar=no,status=no,directories=no,menubar=no,resizable=yes,scrollable=no", true);
    }
    else {
      if (modified == '1') originalbaseURL = originalbaseURL + originalsearchParam;
      openPopup(originalbaseURL, "lookup", 350, 480, "width="+originalwidth+",height=480,toolbar=no,status=no,directories=no,menubar=no,resizable=yes,scrollable=no", true);
    } 
  }
</script>

<apex:sectionHeader title="Demo"  subtitle="Custom Lookup" />

	<apex:form id="myForm">  
		<apex:PageBlock id="PageBlock">		
			<apex:pageBlockSection columns="1" title="Custom Lookup">
        <apex:inputField id="Account" value="{!contact.AccountId}"  />
			</apex:pageBlockSection>
		</apex:PageBlock>
	</apex:form>
       
</apex:page>

MyCustomLookupController.cls

public with sharing class MyCustomLookupController {
   
  public Contact contact {get;set;}
	
  public MyCustomLookupController() {
    contact = new Contact();
  }
    
}