Sunday, January 28, 2018

Sample Unix Script with Documentation

#! /usr/bin/ksh
# +-----------------------------------------------------------------------+
# | Program ID:    ms.sh
# |
# | Purpose:       Put the data into the production table by:
# |                1. FTPing the file from the vax
# |                2. Converting to uppercase for BOA searches
# |                3. Drop the 'PRV' work table
# |                4. Rename the 'CUR' work table to 'PRV'
# |                5. Create the 'CUR' work table
# |                6. Run SQL*Loader to load the 'CUR' work table from the
# |                   vax file & show errors
# |                7. disable_indexes_nonunique (table_name)
# |                8. disable_constraints (table_name, 'R')
# |                9. disable_constraints (table_name, 'P')
# |               10. truncate production table
# |               11. insert/select from the current week
# |               12. insert/select from the previous week
# |               13. rebuild_indexes_nonunique(table_name)
# |               14. enable_constraints(table_name)
# |               15. print status that shows counts, indexes and constraints
# |
# | Programmer:   Lynn C. Tobias
# | Program Date: 03/23/2003
# |
# | Input File(s): ~/.netrc                  contains ftp macros
# |                 master_list_###.fnl2    from vax
# |                 master_list.upp         input to production tables
# |
# | Table(s) Used: Master_List              production table build
# |                Work_Master_List_PrvWk   work table with last week's data
# |                Work_Master_List_CurWk   work table with this week's data
# |                Xref_Period               to get current period
# |                All_Table                 has table information
# |               
# | Called by:    
# | Calls:         datadict.sql       format Oracle dictionary information
# |                crmscur.sql        Create Master_List current week table
# |                countsv.sql        Count records by version and client
# |                indexsv.sql        Show pertinent index information
# |                cnstrsv.sql        Show primary and foreign key information
# |               
# |                Common_Procs:
# |                   Disable_Indexes_Nonunique
# |                   Disable_Constraints
# |                   Truncate_Tbl
# |
# | Output:        ms_drop_ren_cr_###.lst   spool file for drop/rename/create
# |                steps7_12.lst            spool for for steps 7-12
# |                statms.lst               final status (counts, etc)
# |
# | Revisions:     Prog Date     Version Comment/Change
# |                ---- -------- ------- ---------------------------------+
# |               
# +-----------------------------------------------------------------------+

# +-----------------------------------------------------------------------+
# | 1. FTP this file using the macro in the ~/.netrc file
# +-----------------------------------------------------------------------+

cd $datadir
echo "\$ ms" | ftp -vi lark
cd /db13/strview

# +-----------------------------------------------------------------------+
# | 2. Convert all data to uppercase (easier to search in BOA)
# +-----------------------------------------------------------------------+

dd if=$datadir/master_list_$curper.fnl2 of=$datadir/master_list.upp conv=ucase

# +-----------------------------------------------------------------------+
# | This can only be done once per week so check for the spool file before
# | running the following SQL session.
# |
# | 3-5. Drop the table that's 2 weeks old. Rename the current (which is now
# | 1 week old to 'PRV'.  Create a current table to hold this week's data.               
# +-----------------------------------------------------------------------+

if [ -e ms_drop_ren_cr_$curper.lst ];
then echo "Steps 3-5 have already been run for this week";
else goto nodrop
fi

SqlPlus -s $schema << !

@Datadict
Set Echo On
Set Feedback On

/*-------------------------------------------------------*/
/* Get the period code to put in the spool file name     */
/*-------------------------------------------------------*/

Column PerCd New_Value xPerCd

Select To_Char(Period_Code-2) Percd
From Xref_Period
Where Trunc(Sysdate) Between Trunc(Begin_Date) And Trunc(Closing_Date);

Spool ms_drop_ren_cr_&xPerCd..lst

/*----------------------------------------------------------*/
/* 3. Drop the table with data that's 2 weeks old and check */
/*----------------------------------------------------------*/

Drop Table Work_Master_List_PrvWk;

Select Owner, Table_Name, Tablespace_Name, Pct_Free, Pct_Used
From All_Tables
Where Table_Name Like 'WORK_MASTER_LIST_%'
  And Table_Name Like '%WK';

/*-------------------------------------------------------*/
/* 4. Rename the current week's table to 'PRV'           */
/*-------------------------------------------------------*/

Alter Table Work_Master_List_Curwk
Rename To   Work_Master_List_Prvwk;

Select Owner, Table_Name, Tablespace_Name, Pct_Free, Pct_Used
From All_Tables
Where Table_Name Like 'WORK_MASTER_LIST_%'
  And Table_Name Like '%WK';

/*-------------------------------------------------------*/
/* 5. Create a 'CUR' table for current week's data       */
/*-------------------------------------------------------*/

@CrMsCur

Select Owner, Table_Name, Tablespace_Name, Pct_Free, Pct_Used
From All_Tables
Where Table_Name Like 'WORK_MASTER_LIST_%'
  And Table_Name Like '%WK';

Spool Off

exit
!
nodrop:
# +-----------------------------------------------------------------------+
# | Search for any errors in the file
# +-----------------------------------------------------------------------+

grep "ORA-" ms_drop_ren_cr_$curper.lst

# +-----------------------------------------------------------------------+
# | List the file for the log
# +-----------------------------------------------------------------------+

cat ms_drop_ren_cr_$curper.lst

# +-----------------------------------------------------------------------+
# | 6. Load data from the flat file to the 'CUR' table using the
# | information in the control file.
# +-----------------------------------------------------------------------+

SqlLdr $schema Direct=True Control=ms.ctl Data=$datadir/master_list.upp

# +-----------------------------------------------------------------------+
# | List the '.bad' file is created -- this needs to be checked.
# +-----------------------------------------------------------------------+

ls -ltrc ms.bad

# +-----------------------------------------------------------------------+
# | Display key lines from the '.log' file
# +-----------------------------------------------------------------------+

grep "Rows successfully loaded"           ms.log
grep "Rows not loaded due to data errors" ms.log
grep "Total logical records"              ms.log
grep "Elapsed time"                       ms.log
# +-----------------------------------------------------------------------+
# | Load the production table by disable indexes and constraints, truncating
# | the current data, loading data from the work tables to production.
# | The constraints and indexes are then put back in place
# +-----------------------------------------------------------------------+

SqlPlus -s  $schema  << !

Set Echo On
Set Feedback On
Spool steps7_12

Define SVPart  = MS
Define SVTable = Master_List

/*-------------------------------------------------------*/
/* 7. Disable_indexes_nonunique (Table_name)
/*-------------------------------------------------------*/

Exec Common_Procs.Disable_Indexes_Nonunique ('&SVTable');

/*-------------------------------------------------------*/
/* 8. Disable_Constraints (Table_Name, 'r')
/*-------------------------------------------------------*/

Exec Common_Procs.Disable_Constraints('&SVTable','R');

/*-------------------------------------------------------*/
/* 9. Disable_Constraints (Table_Name, 'p')
/*-------------------------------------------------------*/

Exec Common_Procs.Disable_Constraints('&SVTable','P');

/*-------------------------------------------------------*/
/* 10. Truncate Production Table
/*-------------------------------------------------------*/

Variable Outnum Number;
Execute Common_Procs.Truncate_Tbl ('&SVTable', :outnum);
Prompt
Prompt &SVTable Truncate Success:
Prompt 1 = Done
Prompt 0 = Error
Prompt
Print Outnum

/*-------------------------------------------------------*/
/* 11. Insert/select from the current week
/*-------------------------------------------------------*/

Alter Session Enable Parallel Dml;
Alter Session Set Skip_Unusable_Indexes=True;
Set Feedback On
Set Echo On
host date

Insert /*+Parallel(Master_List,2)*/
              Into Master_List    Nologging
   Select  /*+Parallel(Work_Master_List_Curwk,2)*/ *
                  From Work_Master_List_Curwk;
Commit;
host date

/*-------------------------------------------------------*/
/* 12. Insert/select from the previous week
/*-------------------------------------------------------*/

Insert /*+Parallel(Master_List,2) */
              Into Master_List       Nologging
    Select  /*+Parallel(Work_Master_List_Prvwk,2) */ *
                   From Work_Master_List_Prvwk;
Commit;

Select *
From Master_List
Where Rownum < 11;

host date

Spool Off

/*-------------------------------------------------------*/
/* 13. Rebuild_Indexes_Nonunique(Table_Name)
/*-------------------------------------------------------*/

Exec Common_Procs.Rebuild_Indexes_Nonunique('&SVTable');

/*-------------------------------------------------------*/
/* 14. Enable_Constraints(Table_Name)
/*-------------------------------------------------------*/

Exec Common_Procs.Enable_Constraints('&SVTable');

/*-------------------------------------------------------*/
/* 15. Print status that shows counts, indexes and constraints
/*-------------------------------------------------------*/

@Datadict
Set Echo Off

Define Program = STAT&SVPart..Sql
Define Listing = stat&SVPart..lst
Spool stat&SVPart

@CountSV
@IndexSV
@CnstrSV

Spool Off

exit
!


Wednesday, March 30, 2011

Arithmetic Substitution

->VAR=$(( 1+3*2 ))
->echo $VAR
7

->VAR=$(( (1+3)*2 ))
->echo $VAR
8

Filename Substitution with Wildcards

Wildcards
*  0 or more characters
?  1 character
 
Command typed Meaning
ls *display all files
ls *.shdisplay all shell scripts
ls go*display all files starting with the letters 'go' and any possible combination of other characters afterwards
ls go??display all files starting with 'go' and two unknown characters
ls go[abcd]display all files starting with 'go' and one more character of either 'a', 'b', 'c', or 'd'
ls go[a-d]display all files starting with 'go' and one more character in the 'a' to 'd' range
ls go[a-dA-D]display all files starting with 'go' and one more character in the 'a' to 'd' range or the 'A' to 'D' range
ls go[!a-d]display all files starting with 'go' and one more character not in the 'a' to 'd' range
ls [a-d]*display all files starting with 'a', 'b', 'c', and 'd' and any other character -- or no other characters

All About Variables

Format
VARIABLE_NAME = value
echo $VARIABLE_NAME  
Types
  • scalar variable -- holds 1 value at a time (aka name value pairs)
  • array variable -- holds 0 or more values
Variable Names
  • a-z, 0-9 or underscore
  • conventionally uppercase
  • start with letter
Setting Values

PET=fish
PET='fish bowl'
PET="fish bowl"

Setting Scalar Values

->PET= cat #A space will cause an error
ksh: cat:  not found

->PET=cat
->echo $PET
cat

->pet='cat fish'
->echo $pet
cat fish

->PET=ferret
->echo $pet   # $pet is not the same as $PET
cat fish  
->echo $PET
ferret

Setting Array Values

->PET[1]=FERRET
->PET[2]=parrot
->echo ${PET[1]}
FERRET
->echo ${PET[2]}
parrot
->echo ${PET[0]}# set since name is same as scalar variable
ferret

Set Variables to Read-only

->readonly PET # now PET can't be set to a value
->PET=dog
ksh: PET: is read only

Unset Variables (if not readonly)

->TREE=oak
->echo $TREE
oak
->unset TREE
->echo $TREE

Variable Types
  local
available within the current instance. not available to programs started by the shell
  environment
variable available to child processes Use export to create an environmental variable from a local variable                         export VARIABLE=value
  shell
special variables set by the shell

Miscellaneous Unix Commands

type two commands at a timecmd1 ; cmd2
spell check a filecat filename | spell
Reset terminal: if mistakenly viewed a binary file with catstty sane
append results of a command onto an existing filecmd>>file.ext
write results of a command to a filecmd>file.ext
logout[Ctrl]+D or exit
ftp file to host1 from host2 ftp host1
search files for a string
-# = display '#' lines on both sides
-c = display a count
-v = display non-matching lines
-i = ignore case
grep string filename

View File Content

view file contentsmore file.ext
view last 10 lines of a filetail filename
view multiple fileshead filenam* | more
view top # lines of a filehead -# filename
view top 10 lines of a filehead filename

Sort

sort filesort filein > fileout
sort file and remove duplicatessort filein > fileout | uniq
sort file and view duplicatessort filein > fileout | uniq -d
sort file numericallysort -n filein > fileout
sort multiple files togethersort filein1 filein2 > fileout