Requirement:
For an in-house built business intelligence tool, had a requirement to export SharePoint list data into SQL Server table. As directly querying SharePoint content databases is not supported by Microsoft and it also could cause performance issues to your SharePoint farm - Lets not think about it further! Of course, there are 3rd party products available on the market to do this, however PowerShell can be utilized to fulfill this requirement.
This PowerShell script retrieves given SharePoint List(s), iterate through all available fields and dynamically creates SQL Server Table and then inserts the data after translating it, and inserts into the SQL Server database table.
Here is my PowerShell script to Convert SharePoint list into SQL Server table and extract data from SharePoint to SQL:
PowerShell Script to Extract SharePoint List Data into SQL Server Table:
Credits: My sincere thanks to http://www.terrymarshall.com.au/Blog/tabid/162/EntryId/164/SharePoint-2010-Exporting-Lists-to-SQL-Server-Part-1.aspx![]()
For an in-house built business intelligence tool, had a requirement to export SharePoint list data into SQL Server table. As directly querying SharePoint content databases is not supported by Microsoft and it also could cause performance issues to your SharePoint farm - Lets not think about it further! Of course, there are 3rd party products available on the market to do this, however PowerShell can be utilized to fulfill this requirement.
This PowerShell script retrieves given SharePoint List(s), iterate through all available fields and dynamically creates SQL Server Table and then inserts the data after translating it, and inserts into the SQL Server database table.
Here is my PowerShell script to Convert SharePoint list into SQL Server table and extract data from SharePoint to SQL:
PowerShell Script to Extract SharePoint List Data into SQL Server Table:
Add-PSSnapin Microsoft.SharePoint.PowerShell -ErrorAction SilentlyContinueMake sure you have the database created (in my case, its "SharePointBI") and the User Name provided in the script has DBO or similar access on the database before running the script.
Function Create-List2Table($ListName,$TableName, $WebURL)
{
#Configuration Variables
$DatabaseServer="SP16-SQLServer01"
$DatabaseName="SharePointBI"
#SQL Authentication
$UserID="UserID"
$Password="Password"
#Log File
$CurrentDir=Split-Path -Parent -Path $MyInvocation.MyCommand.Definition
$LogFile = $CurrentDir+ "\" + $(Get-Date -format "yyyyMMdd_hhmmss")+".txt"
#Get Web, List and Fields
$Web= Get-SPWeb $WebURL
$List= $Web.Lists[$ListName]
#Get all required fields from the lists
$ListFields = $List.Fields | Where { ($_.Hidden -ne $true ) -and ($_.ReadOnlyField -ne $true ) -and ($_.InternalName -ne "Attachments") -and ($_.InternalName -ne "ContentType") }
#Function to Execute given SQL Query
Function Execute-SQLQuery([string]$Query)
{
#Write-Host "Executing SQL: ", $Query
#Connection object
$Connection = New-Object System.Data.SqlClient.SqlConnection("server=$DatabaseServer;database=$DatabaseName;User Id=$UserID;Password=$Password")
try
{
$Connection.open()
$cmd = new-object System.Data.SqlClient.SqlCommand ($Query, $Connection)
$ReturnValue = $cmd.ExecuteNonQuery()
}
catch { Write-Host "Error: ", $error[0]; "$($Query): $($_.Exception.Message)" >> $LogFile }
finally{ $Connection.close() }
}
#Function to Drop Table, if exists!
Function Drop-Table([string]$TableName)
{
$Query = "IF (OBJECT_ID('[dbo].[$($TableName)]','U') IS NOT NULL) DROP TABLE [dbo].[$($TableName)]"
#Run the Query
Execute-SQLQuery $Query
}
#Get SQL column Definition for SharePoint List Field
Function Get-ColumnDefinition([Microsoft.SharePoint.SPField]$Field)
{
$ColumnDefinition=""
Switch($Field.Type)
{
"Boolean" { $ColumnDefinition = '['+ $Field.InternalName +'] [bit] NULL '}
"Choice" { $ColumnDefinition = '['+ $Field.InternalName +'] [nvarchar](MAX) NULL '}
"Currency" { $ColumnDefinition = '['+ $Field.InternalName +'] [decimal](18, 2) NULL '}
"DateTime" { $ColumnDefinition = '['+ $Field.InternalName +'] [datetime] NULL '}
"Guid" { $ColumnDefinition = '['+ $Field.InternalName +'] [uniqueidentifier] NULL '}
"Integer" { $ColumnDefinition = '['+ $Field.InternalName +'] [int] NULL '}
"Lookup" { $ColumnDefinition = '['+ $Field.InternalName +'] [nvarchar] (500) NULL '}
"MultiChoice" { $ColumnDefinition = '['+ $Field.InternalName +'] [nText] (MAX) NULL '}
"Note" { $ColumnDefinition = '['+ $Field.InternalName +'] [nText] NULL '}
"Number" { $ColumnDefinition = '['+ $Field.InternalName +'] [decimal](18, 2) NULL '}
"Text" { $ColumnDefinition = '['+ $Field.InternalName +'] [nVarchar] (MAX) NULL '}
"URL" { $ColumnDefinition = '['+ $Field.InternalName +'] [nvarchar] (500) NULL '}
"User" { $ColumnDefinition = '['+ $Field.InternalName +'] [nvarchar] (255) NULL '}
default { $ColumnDefinition = '['+ $Field.InternalName +'] [nvarchar] (MAX) NULL '}
}
return $ColumnDefinition
}
################ Format Column Value Functions ######################
Function Format-UserValue([object] $ValueToFormat)
{
$Users = [string]::join("; ",( $ValueToFormat | Select -expandproperty LookupValue))
$Users = $Users -replace "'", "''"
return "'" + $Users + "'"
}
Function Format-LookupValue([Microsoft.SharePoint.SPFieldLookupValueCollection] $ValueToFormat)
{
$LookupValue = [string]::join("; ",( $ValueToFormat | Select -expandproperty LookupValue))
$LookupValue = $LookupValue -replace "'", "''"
return "'" + $LookupValue + "'"
}
Function Format-DateValue([string]$ValueToFormat)
{
[datetime] $dt = $ValueToFormat
return "'" + $dt.ToString("yyyyMMdd HH:MM:ss") + "'"
}
Function Format-MMSValue([Object]$ValueToFormat)
{
return "'" + $ValueToFormat.Label + "'"
}
Function Format-BooleanValue([string]$ValueToFormat)
{
if($ValueToFormat -eq "Yes") {return 1} else { return 0}
}
Function Format-StringValue([object]$ValueToFormat)
{
[string]$result = $ValueToFormat -replace "'", "''"
return "'" + $result + "'"
}
#Function to get the value of given field of the List item
Function Get-ColumnValue([Microsoft.SharePoint.SPListItem] $ListItem, [Microsoft.SharePoint.SPField]$Field)
{
$FieldValue= $ListItem[$Field.InternalName]
#Check for NULL
if([string]::IsNullOrEmpty($FieldValue)) { return 'NULL'}
$FormattedValue = ""
Switch($Field.Type)
{
"Boolean" {$FormattedValue = Format-BooleanValue($FieldValue)}
"Choice" {$FormattedValue = Format-StringValue($FieldValue)}
"Currency" {$FormattedValue = $FieldValue}
"DateTime" {$FormattedValue = Format-DateValue($FieldValue)}
"Guid" { $FormattedValue = Format-StringValue($FieldValue)}
"Integer" {$FormattedValue = $FieldValue}
"Lookup" {$FormattedValue = Format-LookupValue($FieldValue) }
"MultiChoice" {$FormattedValue = Format-StringValue($FieldValue)}
"Note" {$FormattedValue = Format-StringValue($Field.GetFieldValueAsText($ListItem[$Field.InternalName]))}
"Number" {$FormattedValue = $FieldValue}
"Text" {$FormattedValue = Format-StringValue($Field.GetFieldValueAsText($ListItem[$Field.InternalName]))}
"URL" {$FormattedValue = Format-StringValue($FieldValue)}
"User" {$FormattedValue = Format-UserValue($FieldValue) }
#Check MMS Field
"Invalid" { if($Field.TypeDisplayName -eq "Managed Metadata") { $FormattedValue = Format-MMSValue($FieldValue) } else { $FormattedValue =Format-StringValue($FieldValue)} }
default {$FormattedValue = Format-StringValue($FieldValue)}
}
Return $FormattedValue
}
#Create SQL Server table for SharePoint List
Function Create-Table([Microsoft.SharePoint.SPList]$List)
{
#Check if the table exists already
$Query="CREATE TABLE [dbo].[$($TableName)]([ID] [int] NOT NULL PRIMARY KEY, "
foreach ($Field in $ListFields)
{
$Query += Get-ColumnDefinition($Field)
$Query += ","
}
$Query += ")"
#Run the Query
Execute-SQLQuery $Query
}
#Insert Data from SharePoint List to SQL Table
Function Insert-Data([Microsoft.SharePoint.SPList]$List)
{
#Iterate through each row from the list
$ListItems= $List.Items # | where {$_["ID"] -eq 820}
#Progress bar counter
$Counter=0
$ListItemCount=$ListItems.Count
Write-host "Total SharePoint List Items to Copy:" $ListItemCount
foreach ($Item in $ListItems)
{
Write-Progress -Activity "Copying SharePoint List Items. Please wait...`n`n" -status "Processing List Item: $($Item['ID'])" -percentComplete ($Counter/$ListItemCount*100)
$sql = new-object System.Text.StringBuilder
[void]$sql.Append("INSERT INTO [dbo].[$($TableName)] ( [ID] ")
$vals = new-object System.Text.StringBuilder
[void]$vals.Append("VALUES ("+ $Item["ID"])
foreach ($Field in $ListFields)
{
[void]$sql.Append(",[$($Field.InternalName)]")
$ColumnValue = Get-ColumnValue $Item $Field
[void]$vals.Append( ","+ $ColumnValue)
}
[void]$sql.Append(") ")
[void]$vals.Append(") ")
#Combine Field and Values
$SQLStatement = $sql.ToString() + $vals.ToString()
#Run the Query
Execute-SQLQuery $SQLStatement
$Counter=$Counter+1;
}
"Total SharePoint List Items Copied: $($ListItemCount)" >> $LogFile
}
#Call functions to export-import SharePoint list to SQL table
Drop-Table $TableName
Create-Table $List
Insert-Data $List
}
#Call the function to Create SQL Server Table from SharePoint List
Create-List2Table -ListName "Projects" -TableName "ProjectData" -WebURL "https://portal.crescent.com/projects"
#Create-List2Table -ListName "Documents" -TableName "Documents" -WebURL "https://portal.crescent.com"
Credits: My sincere thanks to http://www.terrymarshall.com.au/Blog/tabid/162/EntryId/164/SharePoint-2010-Exporting-Lists-to-SQL-Server-Part-1.aspx
